From 8d26fbd9868cd9286f7d1b9bea3a790cd0a01430 Mon Sep 17 00:00:00 2001 From: David Worms Date: Thu, 10 Feb 2022 12:51:03 +0100 Subject: [PATCH 01/19] docs(csv-stringify): nested columns sample --- .../samples/option.columns_nested.js | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 packages/csv-stringify/samples/option.columns_nested.js diff --git a/packages/csv-stringify/samples/option.columns_nested.js b/packages/csv-stringify/samples/option.columns_nested.js new file mode 100644 index 000000000..0b279ac5a --- /dev/null +++ b/packages/csv-stringify/samples/option.columns_nested.js @@ -0,0 +1,26 @@ + +import { stringify } from 'csv-stringify'; +import assert from 'assert'; + +stringify([{ + an_array: [{ + }, { + field_2: '1_val_1' + }], + an_object: { + field_4: '1_val_2' + } +},{ + an_array: [{ + }, { + field_2: '2_val_1' + }], + an_object: { + field_4: '2_val_2' + } +}], {columns: [ + 'an_object.field_4', + 'an_array[1].field_2' +]}, (err, records) => { + assert.equal(records, "1_val_2,1_val_1\n2_val_2,2_val_1\n"); +}); From 6b1239762ddbc8f76c534c0721ec399e9e48239d Mon Sep 17 00:00:00 2001 From: David Worms Date: Wed, 16 Feb 2022 23:27:02 +0100 Subject: [PATCH 02/19] chore: latest dependencies --- demo/browser/package.json | 2 +- demo/cjs/package.json | 8 ++++---- demo/esm/package.json | 2 +- demo/issues-cjs/package.json | 2 +- demo/issues-esm/package.json | 2 +- demo/webpack-ts/package.json | 8 ++++---- demo/webpack/package.json | 6 +++--- packages/csv-generate/package.json | 16 ++++++++-------- packages/csv-parse/package.json | 16 ++++++++-------- packages/csv-stringify/package.json | 18 +++++++++--------- packages/csv/package.json | 16 ++++++++-------- packages/stream-transform/package.json | 16 ++++++++-------- 12 files changed, 56 insertions(+), 56 deletions(-) diff --git a/demo/browser/package.json b/demo/browser/package.json index 4e723e5d4..9bbfa66cb 100644 --- a/demo/browser/package.json +++ b/demo/browser/package.json @@ -20,6 +20,6 @@ "start": "node server.js" }, "dependencies": { - "express": "^4.17.1" + "express": "^4.17.2" } } diff --git a/demo/cjs/package.json b/demo/cjs/package.json index 6a374c5e3..b7900b45d 100644 --- a/demo/cjs/package.json +++ b/demo/cjs/package.json @@ -6,12 +6,12 @@ "type": "commonjs", "private": true, "devDependencies": { - "@types/node": "^16.11.0", + "@types/node": "^17.0.18", "coffeescript": "^2.6.1", - "mocha": "^9.1.3", + "mocha": "^9.2.0", "should": "^13.2.3", - "ts-node": "^10.3.0", - "typescript": "^4.4.4" + "ts-node": "^10.5.0", + "typescript": "^4.5.5" }, "mocha": { "inline-diffs": true, diff --git a/demo/esm/package.json b/demo/esm/package.json index cf20071f0..9df673067 100644 --- a/demo/esm/package.json +++ b/demo/esm/package.json @@ -7,7 +7,7 @@ "private": true, "devDependencies": { "coffeescript": "^2.6.1", - "mocha": "^9.1.3", + "mocha": "^9.2.0", "should": "^13.2.3" }, "mocha": { diff --git a/demo/issues-cjs/package.json b/demo/issues-cjs/package.json index f48054540..707f0e1e8 100644 --- a/demo/issues-cjs/package.json +++ b/demo/issues-cjs/package.json @@ -6,7 +6,7 @@ "private": true, "devDependencies": { "coffeescript": "^2.6.1", - "mocha": "^9.1.3", + "mocha": "^9.2.0", "should": "^13.2.3" }, "mocha": { diff --git a/demo/issues-esm/package.json b/demo/issues-esm/package.json index 4993ad939..4e5892728 100644 --- a/demo/issues-esm/package.json +++ b/demo/issues-esm/package.json @@ -8,7 +8,7 @@ "devDependencies": { "coffeescript": "^2.6.1", "dirname-filename-esm": "^1.1.1", - "mocha": "^9.1.3", + "mocha": "^9.2.0", "should": "^13.2.3" }, "mocha": { diff --git a/demo/webpack-ts/package.json b/demo/webpack-ts/package.json index b865285de..ee4d7649a 100644 --- a/demo/webpack-ts/package.json +++ b/demo/webpack-ts/package.json @@ -16,11 +16,11 @@ "node-polyfill-webpack-plugin": "^1.1.4", "stream-browserify": "^3.0.0", "ts-loader": "^9.2.6", - "typescript": "^4.5.2", - "webpack": "^5.64.1", - "webpack-cli": "^4.9.1" + "typescript": "^4.5.5", + "webpack": "^5.69.0", + "webpack-cli": "^4.9.2" }, "dependencies": { - "http-server": "^14.0.0" + "http-server": "^14.1.0" } } diff --git a/demo/webpack/package.json b/demo/webpack/package.json index 42cd66283..2864d1d08 100644 --- a/demo/webpack/package.json +++ b/demo/webpack/package.json @@ -12,11 +12,11 @@ "author": "", "license": "ISC", "devDependencies": { - "webpack": "^5.64.1", - "webpack-cli": "^4.9.1" + "webpack": "^5.69.0", + "webpack-cli": "^4.9.2" }, "dependencies": { - "http-server": "^14.0.0", + "http-server": "^14.1.0", "stream-browserify": "^3.0.0" } } diff --git a/packages/csv-generate/package.json b/packages/csv-generate/package.json index 5dd112e60..0854a4eeb 100644 --- a/packages/csv-generate/package.json +++ b/packages/csv-generate/package.json @@ -12,20 +12,20 @@ "author": "David Worms (https://www.adaltas.com)", "devDependencies": { "@rollup/plugin-eslint": "^8.0.1", - "@rollup/plugin-node-resolve": "^13.0.6", - "@types/mocha": "^9.0.0", - "@types/node": "^16.11.7", + "@rollup/plugin-node-resolve": "^13.1.3", + "@types/mocha": "^9.1.0", + "@types/node": "^17.0.18", "@types/should": "^13.0.0", "coffeescript": "~2.6.1", "each": "^1.2.2", - "eslint": "^8.2.0", - "mocha": "~9.1.3", - "rollup": "^2.60.0", + "eslint": "^8.9.0", + "mocha": "~9.2.0", + "rollup": "^2.67.2", "rollup-plugin-node-builtins": "^2.1.2", "rollup-plugin-node-globals": "^1.4.0", "should": "~13.2.3", - "ts-node": "^10.4.0", - "typescript": "^4.4.4" + "ts-node": "^10.5.0", + "typescript": "^4.5.5" }, "exports": { ".": { diff --git a/packages/csv-parse/package.json b/packages/csv-parse/package.json index 04a051ecf..7448cdf56 100644 --- a/packages/csv-parse/package.json +++ b/packages/csv-parse/package.json @@ -43,24 +43,24 @@ }, "devDependencies": { "@rollup/plugin-eslint": "^8.0.1", - "@rollup/plugin-node-resolve": "^13.0.6", - "@types/mocha": "^9.0.0", - "@types/node": "^16.11.7", + "@rollup/plugin-node-resolve": "^13.1.3", + "@types/mocha": "^9.1.0", + "@types/node": "^17.0.18", "coffeelint": "^2.1.0", "coffeescript": "^2.6.1", "csv-generate": "^4.0.4", "csv-spectrum": "^1.0.0", "each": "^1.2.2", - "eslint": "^8.2.0", - "mocha": "^9.1.3", + "eslint": "^8.9.0", + "mocha": "^9.2.0", "pad": "^3.2.0", - "rollup": "^2.60.0", + "rollup": "^2.67.2", "rollup-plugin-node-builtins": "^2.1.2", "rollup-plugin-node-globals": "^1.4.0", "should": "^13.2.3", "stream-transform": "^3.0.4", - "ts-node": "^10.4.0", - "typescript": "^4.4.4" + "ts-node": "^10.5.0", + "typescript": "^4.5.5" }, "files": [ "dist", diff --git a/packages/csv-stringify/package.json b/packages/csv-stringify/package.json index 41c7e9c31..c7b8cbd13 100644 --- a/packages/csv-stringify/package.json +++ b/packages/csv-stringify/package.json @@ -10,22 +10,22 @@ "author": "David Worms (https://www.adaltas.com)", "devDependencies": { "@rollup/plugin-eslint": "^8.0.1", - "@rollup/plugin-node-resolve": "^13.0.6", - "@types/mocha": "^9.0.0", - "@types/node": "^16.11.7", + "@rollup/plugin-node-resolve": "^13.1.3", + "@types/mocha": "^9.1.0", + "@types/node": "^17.0.18", "@types/should": "^13.0.0", "coffeescript": "~2.6.1", "csv-generate": "^4.0.4", "each": "^1.2.2", - "eslint": "^8.2.0", - "express": "^4.17.1", - "mocha": "~9.1.3", - "rollup": "^2.60.0", + "eslint": "^8.9.0", + "express": "^4.17.2", + "mocha": "~9.2.0", + "rollup": "^2.67.2", "rollup-plugin-node-builtins": "^2.1.2", "rollup-plugin-node-globals": "^1.4.0", "should": "~13.2.3", - "ts-node": "^10.4.0", - "typescript": "^4.4.4" + "ts-node": "^10.5.0", + "typescript": "^4.5.5" }, "exports": { ".": { diff --git a/packages/csv/package.json b/packages/csv/package.json index a529f3614..82c1e7e2f 100644 --- a/packages/csv/package.json +++ b/packages/csv/package.json @@ -28,19 +28,19 @@ }, "devDependencies": { "@rollup/plugin-eslint": "^8.0.1", - "@rollup/plugin-node-resolve": "^13.0.6", - "@types/mocha": "^9.0.0", - "@types/node": "^16.11.7", + "@rollup/plugin-node-resolve": "^13.1.3", + "@types/mocha": "^9.1.0", + "@types/node": "^17.0.18", "@types/should": "^13.0.0", "coffeescript": "~2.6.1", - "eslint": "^8.2.0", - "mocha": "~9.1.3", - "rollup": "^2.60.0", + "eslint": "^8.9.0", + "mocha": "~9.2.0", + "rollup": "^2.67.2", "rollup-plugin-node-builtins": "^2.1.2", "rollup-plugin-node-globals": "^1.4.0", "should": "~13.2.3", - "ts-node": "^10.4.0", - "typescript": "^4.4.4" + "ts-node": "^10.5.0", + "typescript": "^4.5.5" }, "engines": { "node": ">= 0.1.90" diff --git a/packages/stream-transform/package.json b/packages/stream-transform/package.json index e29666835..af19ee657 100644 --- a/packages/stream-transform/package.json +++ b/packages/stream-transform/package.json @@ -11,21 +11,21 @@ "author": "David Worms (https://www.adaltas.com)", "devDependencies": { "@rollup/plugin-eslint": "^8.0.1", - "@rollup/plugin-node-resolve": "^13.0.6", - "@types/mocha": "^9.0.0", - "@types/node": "^16.11.7", + "@rollup/plugin-node-resolve": "^13.1.3", + "@types/mocha": "^9.1.0", + "@types/node": "^17.0.18", "coffeescript": "~2.6.1", "csv-generate": "^4.0.4", "each": "^1.2.2", - "eslint": "^8.2.0", - "mocha": "~9.1.3", + "eslint": "^8.9.0", + "mocha": "~9.2.0", "pad": "~3.2.0", - "rollup": "^2.60.0", + "rollup": "^2.67.2", "rollup-plugin-node-builtins": "^2.1.2", "rollup-plugin-node-globals": "^1.4.0", "should": "~13.2.3", - "ts-node": "^10.4.0", - "typescript": "^4.4.4" + "ts-node": "^10.5.0", + "typescript": "^4.5.5" }, "exports": { ".": { From e7372656dd7f9da28c76269f0f0c6bbbfb7e6ea9 Mon Sep 17 00:00:00 2001 From: David Worms Date: Wed, 16 Feb 2022 23:27:43 +0100 Subject: [PATCH 03/19] test: simplify esm loaders --- demo/esm/package.json | 4 +- demo/esm/test/loaders/all.js | 16 ++++++++ demo/esm/test/loaders/all.mjs | 26 ------------- demo/esm/test/loaders/coffee.js | 13 +++++++ demo/esm/test/loaders/coffee.mjs | 36 ------------------ .../esm/test/loaders/legacy/all.js | 2 +- .../loaders/legacy/{coffee.mjs => coffee.js} | 0 demo/issues-esm/package.json | 2 +- demo/issues-esm/test/loaders/all.js | 16 ++++++++ demo/issues-esm/test/loaders/all.mjs | 26 ------------- demo/issues-esm/test/loaders/coffee.js | 13 +++++++ demo/issues-esm/test/loaders/coffee.mjs | 36 ------------------ .../test/loaders/legacy/{all.mjs => all.js} | 2 +- .../loaders/legacy/{coffee.mjs => coffee.js} | 0 packages/csv-generate/package.json | 4 +- packages/csv-generate/test/loaders/all.js | 16 ++++++++ packages/csv-generate/test/loaders/all.mjs | 26 ------------- packages/csv-generate/test/loaders/coffee.js | 13 +++++++ packages/csv-generate/test/loaders/coffee.mjs | 36 ------------------ .../csv-generate/test/loaders/legacy/all.js | 2 +- .../loaders/legacy/{coffee.mjs => coffee.js} | 0 packages/csv-parse/package.json | 4 +- packages/csv-parse/test/loaders/all.js | 16 ++++++++ packages/csv-parse/test/loaders/all.mjs | 26 ------------- packages/csv-parse/test/loaders/coffee.js | 13 +++++++ packages/csv-parse/test/loaders/coffee.mjs | 36 ------------------ .../test/loaders/legacy/{all.mjs => all.js} | 2 +- .../loaders/legacy/{coffee.mjs => coffee.js} | 0 packages/csv-stringify/package.json | 4 +- packages/csv-stringify/test/loaders/all.js | 16 ++++++++ packages/csv-stringify/test/loaders/all.mjs | 26 ------------- packages/csv-stringify/test/loaders/coffee.js | 13 +++++++ .../csv-stringify/test/loaders/coffee.mjs | 36 ------------------ .../csv-stringify/test/loaders/legacy/all.js | 37 +++++++++++++++++++ .../csv-stringify/test/loaders/legacy/all.mjs | 37 ------------------- .../loaders/legacy/{coffee.mjs => coffee.js} | 0 packages/csv/package.json | 4 +- packages/csv/test/loaders/all.js | 16 ++++++++ packages/csv/test/loaders/all.mjs | 26 ------------- packages/csv/test/loaders/coffee.js | 13 +++++++ packages/csv/test/loaders/coffee.mjs | 36 ------------------ packages/csv/test/loaders/legacy/all.js | 37 +++++++++++++++++++ packages/csv/test/loaders/legacy/all.mjs | 37 ------------------- .../loaders/legacy/{coffee.mjs => coffee.js} | 0 packages/stream-transform/package.json | 4 +- packages/stream-transform/test/loaders/all.js | 16 ++++++++ .../stream-transform/test/loaders/all.mjs | 26 ------------- .../stream-transform/test/loaders/coffee.js | 13 +++++++ .../stream-transform/test/loaders/coffee.mjs | 36 ------------------ .../test/loaders/legacy/all.js | 37 +++++++++++++++++++ .../test/loaders/legacy/all.mjs | 37 ------------------- .../loaders/legacy/{coffee.mjs => coffee.js} | 0 52 files changed, 331 insertions(+), 562 deletions(-) create mode 100644 demo/esm/test/loaders/all.js delete mode 100644 demo/esm/test/loaders/all.mjs create mode 100644 demo/esm/test/loaders/coffee.js delete mode 100644 demo/esm/test/loaders/coffee.mjs rename packages/csv-generate/test/loaders/legacy/all.mjs => demo/esm/test/loaders/legacy/all.js (96%) rename demo/esm/test/loaders/legacy/{coffee.mjs => coffee.js} (100%) create mode 100644 demo/issues-esm/test/loaders/all.js delete mode 100644 demo/issues-esm/test/loaders/all.mjs create mode 100644 demo/issues-esm/test/loaders/coffee.js delete mode 100644 demo/issues-esm/test/loaders/coffee.mjs rename demo/issues-esm/test/loaders/legacy/{all.mjs => all.js} (96%) rename demo/issues-esm/test/loaders/legacy/{coffee.mjs => coffee.js} (100%) create mode 100644 packages/csv-generate/test/loaders/all.js delete mode 100644 packages/csv-generate/test/loaders/all.mjs create mode 100644 packages/csv-generate/test/loaders/coffee.js delete mode 100644 packages/csv-generate/test/loaders/coffee.mjs rename demo/esm/test/loaders/legacy/all.mjs => packages/csv-generate/test/loaders/legacy/all.js (96%) rename packages/csv-generate/test/loaders/legacy/{coffee.mjs => coffee.js} (100%) create mode 100644 packages/csv-parse/test/loaders/all.js delete mode 100644 packages/csv-parse/test/loaders/all.mjs create mode 100644 packages/csv-parse/test/loaders/coffee.js delete mode 100644 packages/csv-parse/test/loaders/coffee.mjs rename packages/csv-parse/test/loaders/legacy/{all.mjs => all.js} (96%) rename packages/csv-parse/test/loaders/legacy/{coffee.mjs => coffee.js} (100%) create mode 100644 packages/csv-stringify/test/loaders/all.js delete mode 100644 packages/csv-stringify/test/loaders/all.mjs create mode 100644 packages/csv-stringify/test/loaders/coffee.js delete mode 100644 packages/csv-stringify/test/loaders/coffee.mjs create mode 100644 packages/csv-stringify/test/loaders/legacy/all.js delete mode 100644 packages/csv-stringify/test/loaders/legacy/all.mjs rename packages/csv-stringify/test/loaders/legacy/{coffee.mjs => coffee.js} (100%) create mode 100644 packages/csv/test/loaders/all.js delete mode 100644 packages/csv/test/loaders/all.mjs create mode 100644 packages/csv/test/loaders/coffee.js delete mode 100644 packages/csv/test/loaders/coffee.mjs create mode 100644 packages/csv/test/loaders/legacy/all.js delete mode 100644 packages/csv/test/loaders/legacy/all.mjs rename packages/csv/test/loaders/legacy/{coffee.mjs => coffee.js} (100%) create mode 100644 packages/stream-transform/test/loaders/all.js delete mode 100644 packages/stream-transform/test/loaders/all.mjs create mode 100644 packages/stream-transform/test/loaders/coffee.js delete mode 100644 packages/stream-transform/test/loaders/coffee.mjs create mode 100644 packages/stream-transform/test/loaders/legacy/all.js delete mode 100644 packages/stream-transform/test/loaders/legacy/all.mjs rename packages/stream-transform/test/loaders/legacy/{coffee.mjs => coffee.js} (100%) diff --git a/demo/esm/package.json b/demo/esm/package.json index 9df673067..b24abf35a 100644 --- a/demo/esm/package.json +++ b/demo/esm/package.json @@ -12,7 +12,7 @@ }, "mocha": { "inline-diffs": true, - "loader": "./test/loaders/all.mjs", + "loader": "./test/loaders/all.js", "recursive": true, "reporter": "spec", "require": [ @@ -23,6 +23,6 @@ }, "scripts": { "test": "mocha 'test/**/*.coffee'", - "test:legacy": "mocha --loader=./test/loaders/legacy/all.mjs 'test/**/*.{coffee,ts}'" + "test:legacy": "mocha --loader=./test/loaders/legacy/all.js 'test/**/*.{coffee,ts}'" } } diff --git a/demo/esm/test/loaders/all.js b/demo/esm/test/loaders/all.js new file mode 100644 index 000000000..344154640 --- /dev/null +++ b/demo/esm/test/loaders/all.js @@ -0,0 +1,16 @@ + +import * as coffee from './coffee.js' +import * as ts from 'ts-node/esm' + +const coffeeRegex = /\.coffee$|\.litcoffee$|\.coffee\.md$/; +const tsRegex = /\.ts$/; + +export function load(url, context, next) { + if (coffeeRegex.test(url)) { + return coffee.load.apply(this, arguments) + } + if (tsRegex.test(url)) { + return ts.load.apply(this, arguments) + } + return next(url, context, next); +} diff --git a/demo/esm/test/loaders/all.mjs b/demo/esm/test/loaders/all.mjs deleted file mode 100644 index affe659fd..000000000 --- a/demo/esm/test/loaders/all.mjs +++ /dev/null @@ -1,26 +0,0 @@ - -import * as coffee from './coffee.mjs' -import * as ts from 'ts-node/esm' - -const coffeeRegex = /\.coffee$|\.litcoffee$|\.coffee\.md$/; -const tsRegex = /\.ts$/; - -export async function resolve(specifier, context, defaultResolve) { - if (coffeeRegex.test(specifier)) { - return coffee.resolve.apply(this, arguments) - } - if (tsRegex.test(specifier)) { - return ts.resolve.apply(this, arguments) - } - return defaultResolve(specifier, context, defaultResolve); -} - -export function load(url, context, defaultLoad) { - if (coffeeRegex.test(url)) { - return coffee.load.apply(this, arguments) - } - if (tsRegex.test(url)) { - return ts.load.apply(this, arguments) - } - return defaultLoad(url, context, defaultLoad); -} diff --git a/demo/esm/test/loaders/coffee.js b/demo/esm/test/loaders/coffee.js new file mode 100644 index 000000000..1252cdb81 --- /dev/null +++ b/demo/esm/test/loaders/coffee.js @@ -0,0 +1,13 @@ +import CoffeeScript from 'coffeescript'; + +const coffeeRegex = /\.coffee$|\.litcoffee$|\.coffee\.md$/; + +export async function load(url, context, next) { + if (coffeeRegex.test(url)) { + const format = 'module'; + const { source: rawSource } = await next(url, { format }); + const source = CoffeeScript.compile(rawSource.toString(), { bare: true }); + return {format, source}; + } + return next(url, context); +} diff --git a/demo/esm/test/loaders/coffee.mjs b/demo/esm/test/loaders/coffee.mjs deleted file mode 100644 index f54674cf6..000000000 --- a/demo/esm/test/loaders/coffee.mjs +++ /dev/null @@ -1,36 +0,0 @@ -// coffeescript-loader.mjs -import { URL, pathToFileURL } from 'url'; -import CoffeeScript from 'coffeescript'; -import { cwd } from 'process'; - -const baseURL = pathToFileURL(`${cwd()}/`).href; - -// CoffeeScript files end in .coffee, .litcoffee or .coffee.md. -const extensionsRegex = /\.coffee$|\.litcoffee$|\.coffee\.md$/; - -export function resolve(specifier, context, defaultResolve) { - const { parentURL = baseURL } = context; - // Node.js normally errors on unknown file extensions, so return a URL for - // specifiers ending in the CoffeeScript file extensions. - if (extensionsRegex.test(specifier)) { - return { - url: new URL(specifier, parentURL).href - }; - } - return defaultResolve(specifier, context, defaultResolve); -} - -export async function load(url, context, defaultLoad) { - if (extensionsRegex.test(url)) { - const format = 'module' - const { source: rawSource } = await defaultLoad(url, { format }); - return { - format: 'module', - source: CoffeeScript.compile(rawSource.toString(), { - bare: true, - filename: url, - }) - }; - } - return defaultLoad(url, context, defaultLoad); -} diff --git a/packages/csv-generate/test/loaders/legacy/all.mjs b/demo/esm/test/loaders/legacy/all.js similarity index 96% rename from packages/csv-generate/test/loaders/legacy/all.mjs rename to demo/esm/test/loaders/legacy/all.js index 4a3828589..c95db0b56 100644 --- a/packages/csv-generate/test/loaders/legacy/all.mjs +++ b/demo/esm/test/loaders/legacy/all.js @@ -1,5 +1,5 @@ -import * as coffee from './coffee.mjs' +import * as coffee from './coffee.js' import * as ts from 'ts-node/esm' const coffeeRegex = /\.coffee$|\.litcoffee$|\.coffee\.md$/; diff --git a/demo/esm/test/loaders/legacy/coffee.mjs b/demo/esm/test/loaders/legacy/coffee.js similarity index 100% rename from demo/esm/test/loaders/legacy/coffee.mjs rename to demo/esm/test/loaders/legacy/coffee.js diff --git a/demo/issues-esm/package.json b/demo/issues-esm/package.json index 4e5892728..ed90dd0de 100644 --- a/demo/issues-esm/package.json +++ b/demo/issues-esm/package.json @@ -13,7 +13,7 @@ }, "mocha": { "inline-diffs": true, - "loader": "./test/loaders/all.mjs", + "loader": "./test/loaders/all.js", "recursive": true, "reporter": "spec", "require": [ diff --git a/demo/issues-esm/test/loaders/all.js b/demo/issues-esm/test/loaders/all.js new file mode 100644 index 000000000..344154640 --- /dev/null +++ b/demo/issues-esm/test/loaders/all.js @@ -0,0 +1,16 @@ + +import * as coffee from './coffee.js' +import * as ts from 'ts-node/esm' + +const coffeeRegex = /\.coffee$|\.litcoffee$|\.coffee\.md$/; +const tsRegex = /\.ts$/; + +export function load(url, context, next) { + if (coffeeRegex.test(url)) { + return coffee.load.apply(this, arguments) + } + if (tsRegex.test(url)) { + return ts.load.apply(this, arguments) + } + return next(url, context, next); +} diff --git a/demo/issues-esm/test/loaders/all.mjs b/demo/issues-esm/test/loaders/all.mjs deleted file mode 100644 index affe659fd..000000000 --- a/demo/issues-esm/test/loaders/all.mjs +++ /dev/null @@ -1,26 +0,0 @@ - -import * as coffee from './coffee.mjs' -import * as ts from 'ts-node/esm' - -const coffeeRegex = /\.coffee$|\.litcoffee$|\.coffee\.md$/; -const tsRegex = /\.ts$/; - -export async function resolve(specifier, context, defaultResolve) { - if (coffeeRegex.test(specifier)) { - return coffee.resolve.apply(this, arguments) - } - if (tsRegex.test(specifier)) { - return ts.resolve.apply(this, arguments) - } - return defaultResolve(specifier, context, defaultResolve); -} - -export function load(url, context, defaultLoad) { - if (coffeeRegex.test(url)) { - return coffee.load.apply(this, arguments) - } - if (tsRegex.test(url)) { - return ts.load.apply(this, arguments) - } - return defaultLoad(url, context, defaultLoad); -} diff --git a/demo/issues-esm/test/loaders/coffee.js b/demo/issues-esm/test/loaders/coffee.js new file mode 100644 index 000000000..1252cdb81 --- /dev/null +++ b/demo/issues-esm/test/loaders/coffee.js @@ -0,0 +1,13 @@ +import CoffeeScript from 'coffeescript'; + +const coffeeRegex = /\.coffee$|\.litcoffee$|\.coffee\.md$/; + +export async function load(url, context, next) { + if (coffeeRegex.test(url)) { + const format = 'module'; + const { source: rawSource } = await next(url, { format }); + const source = CoffeeScript.compile(rawSource.toString(), { bare: true }); + return {format, source}; + } + return next(url, context); +} diff --git a/demo/issues-esm/test/loaders/coffee.mjs b/demo/issues-esm/test/loaders/coffee.mjs deleted file mode 100644 index f54674cf6..000000000 --- a/demo/issues-esm/test/loaders/coffee.mjs +++ /dev/null @@ -1,36 +0,0 @@ -// coffeescript-loader.mjs -import { URL, pathToFileURL } from 'url'; -import CoffeeScript from 'coffeescript'; -import { cwd } from 'process'; - -const baseURL = pathToFileURL(`${cwd()}/`).href; - -// CoffeeScript files end in .coffee, .litcoffee or .coffee.md. -const extensionsRegex = /\.coffee$|\.litcoffee$|\.coffee\.md$/; - -export function resolve(specifier, context, defaultResolve) { - const { parentURL = baseURL } = context; - // Node.js normally errors on unknown file extensions, so return a URL for - // specifiers ending in the CoffeeScript file extensions. - if (extensionsRegex.test(specifier)) { - return { - url: new URL(specifier, parentURL).href - }; - } - return defaultResolve(specifier, context, defaultResolve); -} - -export async function load(url, context, defaultLoad) { - if (extensionsRegex.test(url)) { - const format = 'module' - const { source: rawSource } = await defaultLoad(url, { format }); - return { - format: 'module', - source: CoffeeScript.compile(rawSource.toString(), { - bare: true, - filename: url, - }) - }; - } - return defaultLoad(url, context, defaultLoad); -} diff --git a/demo/issues-esm/test/loaders/legacy/all.mjs b/demo/issues-esm/test/loaders/legacy/all.js similarity index 96% rename from demo/issues-esm/test/loaders/legacy/all.mjs rename to demo/issues-esm/test/loaders/legacy/all.js index 4a3828589..c95db0b56 100644 --- a/demo/issues-esm/test/loaders/legacy/all.mjs +++ b/demo/issues-esm/test/loaders/legacy/all.js @@ -1,5 +1,5 @@ -import * as coffee from './coffee.mjs' +import * as coffee from './coffee.js' import * as ts from 'ts-node/esm' const coffeeRegex = /\.coffee$|\.litcoffee$|\.coffee\.md$/; diff --git a/demo/issues-esm/test/loaders/legacy/coffee.mjs b/demo/issues-esm/test/loaders/legacy/coffee.js similarity index 100% rename from demo/issues-esm/test/loaders/legacy/coffee.mjs rename to demo/issues-esm/test/loaders/legacy/coffee.js diff --git a/packages/csv-generate/package.json b/packages/csv-generate/package.json index 0854a4eeb..8795720dc 100644 --- a/packages/csv-generate/package.json +++ b/packages/csv-generate/package.json @@ -48,7 +48,7 @@ "main": "./dist/cjs/index.cjs", "mocha": { "inline-diffs": true, - "loader": "./test/loaders/all.mjs", + "loader": "./test/loaders/all.js", "recursive": true, "reporter": "spec", "require": [ @@ -73,7 +73,7 @@ "preversion": "npm run build && git add dist", "pretest": "npm run build", "test": "mocha 'test/**/*.{coffee,ts}'", - "test:legacy": "mocha --loader=./test/loaders/legacy/all.mjs 'test/**/*.{coffee,ts}'" + "test:legacy": "mocha --loader=./test/loaders/legacy/all.js 'test/**/*.{coffee,ts}'" }, "type": "module", "types": "dist/esm/index.d.ts", diff --git a/packages/csv-generate/test/loaders/all.js b/packages/csv-generate/test/loaders/all.js new file mode 100644 index 000000000..344154640 --- /dev/null +++ b/packages/csv-generate/test/loaders/all.js @@ -0,0 +1,16 @@ + +import * as coffee from './coffee.js' +import * as ts from 'ts-node/esm' + +const coffeeRegex = /\.coffee$|\.litcoffee$|\.coffee\.md$/; +const tsRegex = /\.ts$/; + +export function load(url, context, next) { + if (coffeeRegex.test(url)) { + return coffee.load.apply(this, arguments) + } + if (tsRegex.test(url)) { + return ts.load.apply(this, arguments) + } + return next(url, context, next); +} diff --git a/packages/csv-generate/test/loaders/all.mjs b/packages/csv-generate/test/loaders/all.mjs deleted file mode 100644 index affe659fd..000000000 --- a/packages/csv-generate/test/loaders/all.mjs +++ /dev/null @@ -1,26 +0,0 @@ - -import * as coffee from './coffee.mjs' -import * as ts from 'ts-node/esm' - -const coffeeRegex = /\.coffee$|\.litcoffee$|\.coffee\.md$/; -const tsRegex = /\.ts$/; - -export async function resolve(specifier, context, defaultResolve) { - if (coffeeRegex.test(specifier)) { - return coffee.resolve.apply(this, arguments) - } - if (tsRegex.test(specifier)) { - return ts.resolve.apply(this, arguments) - } - return defaultResolve(specifier, context, defaultResolve); -} - -export function load(url, context, defaultLoad) { - if (coffeeRegex.test(url)) { - return coffee.load.apply(this, arguments) - } - if (tsRegex.test(url)) { - return ts.load.apply(this, arguments) - } - return defaultLoad(url, context, defaultLoad); -} diff --git a/packages/csv-generate/test/loaders/coffee.js b/packages/csv-generate/test/loaders/coffee.js new file mode 100644 index 000000000..1252cdb81 --- /dev/null +++ b/packages/csv-generate/test/loaders/coffee.js @@ -0,0 +1,13 @@ +import CoffeeScript from 'coffeescript'; + +const coffeeRegex = /\.coffee$|\.litcoffee$|\.coffee\.md$/; + +export async function load(url, context, next) { + if (coffeeRegex.test(url)) { + const format = 'module'; + const { source: rawSource } = await next(url, { format }); + const source = CoffeeScript.compile(rawSource.toString(), { bare: true }); + return {format, source}; + } + return next(url, context); +} diff --git a/packages/csv-generate/test/loaders/coffee.mjs b/packages/csv-generate/test/loaders/coffee.mjs deleted file mode 100644 index f54674cf6..000000000 --- a/packages/csv-generate/test/loaders/coffee.mjs +++ /dev/null @@ -1,36 +0,0 @@ -// coffeescript-loader.mjs -import { URL, pathToFileURL } from 'url'; -import CoffeeScript from 'coffeescript'; -import { cwd } from 'process'; - -const baseURL = pathToFileURL(`${cwd()}/`).href; - -// CoffeeScript files end in .coffee, .litcoffee or .coffee.md. -const extensionsRegex = /\.coffee$|\.litcoffee$|\.coffee\.md$/; - -export function resolve(specifier, context, defaultResolve) { - const { parentURL = baseURL } = context; - // Node.js normally errors on unknown file extensions, so return a URL for - // specifiers ending in the CoffeeScript file extensions. - if (extensionsRegex.test(specifier)) { - return { - url: new URL(specifier, parentURL).href - }; - } - return defaultResolve(specifier, context, defaultResolve); -} - -export async function load(url, context, defaultLoad) { - if (extensionsRegex.test(url)) { - const format = 'module' - const { source: rawSource } = await defaultLoad(url, { format }); - return { - format: 'module', - source: CoffeeScript.compile(rawSource.toString(), { - bare: true, - filename: url, - }) - }; - } - return defaultLoad(url, context, defaultLoad); -} diff --git a/demo/esm/test/loaders/legacy/all.mjs b/packages/csv-generate/test/loaders/legacy/all.js similarity index 96% rename from demo/esm/test/loaders/legacy/all.mjs rename to packages/csv-generate/test/loaders/legacy/all.js index 4a3828589..c95db0b56 100644 --- a/demo/esm/test/loaders/legacy/all.mjs +++ b/packages/csv-generate/test/loaders/legacy/all.js @@ -1,5 +1,5 @@ -import * as coffee from './coffee.mjs' +import * as coffee from './coffee.js' import * as ts from 'ts-node/esm' const coffeeRegex = /\.coffee$|\.litcoffee$|\.coffee\.md$/; diff --git a/packages/csv-generate/test/loaders/legacy/coffee.mjs b/packages/csv-generate/test/loaders/legacy/coffee.js similarity index 100% rename from packages/csv-generate/test/loaders/legacy/coffee.mjs rename to packages/csv-generate/test/loaders/legacy/coffee.js diff --git a/packages/csv-parse/package.json b/packages/csv-parse/package.json index 7448cdf56..3074ec110 100644 --- a/packages/csv-parse/package.json +++ b/packages/csv-parse/package.json @@ -71,7 +71,7 @@ "main": "./dist/cjs/index.cjs", "mocha": { "inline-diffs": true, - "loader": "./test/loaders/all.mjs", + "loader": "./test/loaders/all.js", "recursive": true, "reporter": "spec", "require": [ @@ -96,7 +96,7 @@ "preversion": "npm run build && git add dist", "pretest": "npm run build", "test": "mocha 'test/**/*.{coffee,ts}'", - "test:legacy": "mocha --loader=./test/loaders/legacy/all.mjs 'test/**/*.{coffee,ts}'" + "test:legacy": "mocha --loader=./test/loaders/legacy/all.js 'test/**/*.{coffee,ts}'" }, "type": "module", "types": "dist/esm/index.d.ts", diff --git a/packages/csv-parse/test/loaders/all.js b/packages/csv-parse/test/loaders/all.js new file mode 100644 index 000000000..344154640 --- /dev/null +++ b/packages/csv-parse/test/loaders/all.js @@ -0,0 +1,16 @@ + +import * as coffee from './coffee.js' +import * as ts from 'ts-node/esm' + +const coffeeRegex = /\.coffee$|\.litcoffee$|\.coffee\.md$/; +const tsRegex = /\.ts$/; + +export function load(url, context, next) { + if (coffeeRegex.test(url)) { + return coffee.load.apply(this, arguments) + } + if (tsRegex.test(url)) { + return ts.load.apply(this, arguments) + } + return next(url, context, next); +} diff --git a/packages/csv-parse/test/loaders/all.mjs b/packages/csv-parse/test/loaders/all.mjs deleted file mode 100644 index affe659fd..000000000 --- a/packages/csv-parse/test/loaders/all.mjs +++ /dev/null @@ -1,26 +0,0 @@ - -import * as coffee from './coffee.mjs' -import * as ts from 'ts-node/esm' - -const coffeeRegex = /\.coffee$|\.litcoffee$|\.coffee\.md$/; -const tsRegex = /\.ts$/; - -export async function resolve(specifier, context, defaultResolve) { - if (coffeeRegex.test(specifier)) { - return coffee.resolve.apply(this, arguments) - } - if (tsRegex.test(specifier)) { - return ts.resolve.apply(this, arguments) - } - return defaultResolve(specifier, context, defaultResolve); -} - -export function load(url, context, defaultLoad) { - if (coffeeRegex.test(url)) { - return coffee.load.apply(this, arguments) - } - if (tsRegex.test(url)) { - return ts.load.apply(this, arguments) - } - return defaultLoad(url, context, defaultLoad); -} diff --git a/packages/csv-parse/test/loaders/coffee.js b/packages/csv-parse/test/loaders/coffee.js new file mode 100644 index 000000000..1252cdb81 --- /dev/null +++ b/packages/csv-parse/test/loaders/coffee.js @@ -0,0 +1,13 @@ +import CoffeeScript from 'coffeescript'; + +const coffeeRegex = /\.coffee$|\.litcoffee$|\.coffee\.md$/; + +export async function load(url, context, next) { + if (coffeeRegex.test(url)) { + const format = 'module'; + const { source: rawSource } = await next(url, { format }); + const source = CoffeeScript.compile(rawSource.toString(), { bare: true }); + return {format, source}; + } + return next(url, context); +} diff --git a/packages/csv-parse/test/loaders/coffee.mjs b/packages/csv-parse/test/loaders/coffee.mjs deleted file mode 100644 index f54674cf6..000000000 --- a/packages/csv-parse/test/loaders/coffee.mjs +++ /dev/null @@ -1,36 +0,0 @@ -// coffeescript-loader.mjs -import { URL, pathToFileURL } from 'url'; -import CoffeeScript from 'coffeescript'; -import { cwd } from 'process'; - -const baseURL = pathToFileURL(`${cwd()}/`).href; - -// CoffeeScript files end in .coffee, .litcoffee or .coffee.md. -const extensionsRegex = /\.coffee$|\.litcoffee$|\.coffee\.md$/; - -export function resolve(specifier, context, defaultResolve) { - const { parentURL = baseURL } = context; - // Node.js normally errors on unknown file extensions, so return a URL for - // specifiers ending in the CoffeeScript file extensions. - if (extensionsRegex.test(specifier)) { - return { - url: new URL(specifier, parentURL).href - }; - } - return defaultResolve(specifier, context, defaultResolve); -} - -export async function load(url, context, defaultLoad) { - if (extensionsRegex.test(url)) { - const format = 'module' - const { source: rawSource } = await defaultLoad(url, { format }); - return { - format: 'module', - source: CoffeeScript.compile(rawSource.toString(), { - bare: true, - filename: url, - }) - }; - } - return defaultLoad(url, context, defaultLoad); -} diff --git a/packages/csv-parse/test/loaders/legacy/all.mjs b/packages/csv-parse/test/loaders/legacy/all.js similarity index 96% rename from packages/csv-parse/test/loaders/legacy/all.mjs rename to packages/csv-parse/test/loaders/legacy/all.js index 4a3828589..c95db0b56 100644 --- a/packages/csv-parse/test/loaders/legacy/all.mjs +++ b/packages/csv-parse/test/loaders/legacy/all.js @@ -1,5 +1,5 @@ -import * as coffee from './coffee.mjs' +import * as coffee from './coffee.js' import * as ts from 'ts-node/esm' const coffeeRegex = /\.coffee$|\.litcoffee$|\.coffee\.md$/; diff --git a/packages/csv-parse/test/loaders/legacy/coffee.mjs b/packages/csv-parse/test/loaders/legacy/coffee.js similarity index 100% rename from packages/csv-parse/test/loaders/legacy/coffee.mjs rename to packages/csv-parse/test/loaders/legacy/coffee.js diff --git a/packages/csv-stringify/package.json b/packages/csv-stringify/package.json index c7b8cbd13..88370f16b 100644 --- a/packages/csv-stringify/package.json +++ b/packages/csv-stringify/package.json @@ -48,7 +48,7 @@ "main": "./dist/cjs/index.cjs", "mocha": { "inline-diffs": true, - "loader": "./test/loaders/all.mjs", + "loader": "./test/loaders/all.js", "recursive": true, "reporter": "spec", "require": [ @@ -73,7 +73,7 @@ "preversion": "npm run build && git add dist", "pretest": "npm run build", "test": "mocha 'test/**/*.{coffee,ts}'", - "test:legacy": "mocha --loader=./test/loaders/legacy/all.mjs 'test/**/*.{coffee,ts}'" + "test:legacy": "mocha --loader=./test/loaders/legacy/all.js 'test/**/*.{coffee,ts}'" }, "type": "module", "types": "dist/esm/index.d.ts", diff --git a/packages/csv-stringify/test/loaders/all.js b/packages/csv-stringify/test/loaders/all.js new file mode 100644 index 000000000..344154640 --- /dev/null +++ b/packages/csv-stringify/test/loaders/all.js @@ -0,0 +1,16 @@ + +import * as coffee from './coffee.js' +import * as ts from 'ts-node/esm' + +const coffeeRegex = /\.coffee$|\.litcoffee$|\.coffee\.md$/; +const tsRegex = /\.ts$/; + +export function load(url, context, next) { + if (coffeeRegex.test(url)) { + return coffee.load.apply(this, arguments) + } + if (tsRegex.test(url)) { + return ts.load.apply(this, arguments) + } + return next(url, context, next); +} diff --git a/packages/csv-stringify/test/loaders/all.mjs b/packages/csv-stringify/test/loaders/all.mjs deleted file mode 100644 index affe659fd..000000000 --- a/packages/csv-stringify/test/loaders/all.mjs +++ /dev/null @@ -1,26 +0,0 @@ - -import * as coffee from './coffee.mjs' -import * as ts from 'ts-node/esm' - -const coffeeRegex = /\.coffee$|\.litcoffee$|\.coffee\.md$/; -const tsRegex = /\.ts$/; - -export async function resolve(specifier, context, defaultResolve) { - if (coffeeRegex.test(specifier)) { - return coffee.resolve.apply(this, arguments) - } - if (tsRegex.test(specifier)) { - return ts.resolve.apply(this, arguments) - } - return defaultResolve(specifier, context, defaultResolve); -} - -export function load(url, context, defaultLoad) { - if (coffeeRegex.test(url)) { - return coffee.load.apply(this, arguments) - } - if (tsRegex.test(url)) { - return ts.load.apply(this, arguments) - } - return defaultLoad(url, context, defaultLoad); -} diff --git a/packages/csv-stringify/test/loaders/coffee.js b/packages/csv-stringify/test/loaders/coffee.js new file mode 100644 index 000000000..1252cdb81 --- /dev/null +++ b/packages/csv-stringify/test/loaders/coffee.js @@ -0,0 +1,13 @@ +import CoffeeScript from 'coffeescript'; + +const coffeeRegex = /\.coffee$|\.litcoffee$|\.coffee\.md$/; + +export async function load(url, context, next) { + if (coffeeRegex.test(url)) { + const format = 'module'; + const { source: rawSource } = await next(url, { format }); + const source = CoffeeScript.compile(rawSource.toString(), { bare: true }); + return {format, source}; + } + return next(url, context); +} diff --git a/packages/csv-stringify/test/loaders/coffee.mjs b/packages/csv-stringify/test/loaders/coffee.mjs deleted file mode 100644 index f54674cf6..000000000 --- a/packages/csv-stringify/test/loaders/coffee.mjs +++ /dev/null @@ -1,36 +0,0 @@ -// coffeescript-loader.mjs -import { URL, pathToFileURL } from 'url'; -import CoffeeScript from 'coffeescript'; -import { cwd } from 'process'; - -const baseURL = pathToFileURL(`${cwd()}/`).href; - -// CoffeeScript files end in .coffee, .litcoffee or .coffee.md. -const extensionsRegex = /\.coffee$|\.litcoffee$|\.coffee\.md$/; - -export function resolve(specifier, context, defaultResolve) { - const { parentURL = baseURL } = context; - // Node.js normally errors on unknown file extensions, so return a URL for - // specifiers ending in the CoffeeScript file extensions. - if (extensionsRegex.test(specifier)) { - return { - url: new URL(specifier, parentURL).href - }; - } - return defaultResolve(specifier, context, defaultResolve); -} - -export async function load(url, context, defaultLoad) { - if (extensionsRegex.test(url)) { - const format = 'module' - const { source: rawSource } = await defaultLoad(url, { format }); - return { - format: 'module', - source: CoffeeScript.compile(rawSource.toString(), { - bare: true, - filename: url, - }) - }; - } - return defaultLoad(url, context, defaultLoad); -} diff --git a/packages/csv-stringify/test/loaders/legacy/all.js b/packages/csv-stringify/test/loaders/legacy/all.js new file mode 100644 index 000000000..c95db0b56 --- /dev/null +++ b/packages/csv-stringify/test/loaders/legacy/all.js @@ -0,0 +1,37 @@ + +import * as coffee from './coffee.js' +import * as ts from 'ts-node/esm' + +const coffeeRegex = /\.coffee$|\.litcoffee$|\.coffee\.md$/; +const tsRegex = /\.ts$/; + +export function resolve(specifier, context, defaultResolve) { + if (coffeeRegex.test(specifier)) { + return coffee.resolve.apply(this, arguments) + } + if (tsRegex.test(specifier)) { + return ts.resolve.apply(this, arguments) + } + return ts.resolve.apply(this, arguments); +} + +export function getFormat(url, context, defaultGetFormat) { + if (coffeeRegex.test(url)) { + return coffee.getFormat.apply(this, arguments) + } + if (tsRegex.test(url)) { + return ts.getFormat.apply(this, arguments) + } + return ts.getFormat.apply(this, arguments); +} + +export function transformSource(source, context, defaultTransformSource) { + const { url } = context; + if (coffeeRegex.test(url)) { + return coffee.transformSource.apply(this, arguments) + } + if (tsRegex.test(url)) { + return ts.transformSource.apply(this, arguments) + } + return ts.transformSource.apply(this, arguments); +} diff --git a/packages/csv-stringify/test/loaders/legacy/all.mjs b/packages/csv-stringify/test/loaders/legacy/all.mjs deleted file mode 100644 index 4a3828589..000000000 --- a/packages/csv-stringify/test/loaders/legacy/all.mjs +++ /dev/null @@ -1,37 +0,0 @@ - -import * as coffee from './coffee.mjs' -import * as ts from 'ts-node/esm' - -const coffeeRegex = /\.coffee$|\.litcoffee$|\.coffee\.md$/; -const tsRegex = /\.ts$/; - -export function resolve(specifier, context, defaultResolve) { - if (coffeeRegex.test(specifier)) { - return coffee.resolve.apply(this, arguments) - } - if (tsRegex.test(specifier)) { - return ts.resolve.apply(this, arguments) - } - return ts.resolve.apply(this, arguments); -} - -export function getFormat(url, context, defaultGetFormat) { - if (coffeeRegex.test(url)) { - return coffee.getFormat.apply(this, arguments) - } - if (tsRegex.test(url)) { - return ts.getFormat.apply(this, arguments) - } - return ts.getFormat.apply(this, arguments); -} - -export function transformSource(source, context, defaultTransformSource) { - const { url } = context; - if (coffeeRegex.test(url)) { - return coffee.transformSource.apply(this, arguments) - } - if (tsRegex.test(url)) { - return ts.transformSource.apply(this, arguments) - } - return ts.transformSource.apply(this, arguments); -} diff --git a/packages/csv-stringify/test/loaders/legacy/coffee.mjs b/packages/csv-stringify/test/loaders/legacy/coffee.js similarity index 100% rename from packages/csv-stringify/test/loaders/legacy/coffee.mjs rename to packages/csv-stringify/test/loaders/legacy/coffee.js diff --git a/packages/csv/package.json b/packages/csv/package.json index 82c1e7e2f..227c736e6 100644 --- a/packages/csv/package.json +++ b/packages/csv/package.json @@ -66,7 +66,7 @@ "main": "./dist/cjs/index.cjs", "mocha": { "inline-diffs": true, - "loader": "./test/loaders/all.mjs", + "loader": "./test/loaders/all.js", "recursive": true, "reporter": "spec", "require": [ @@ -91,7 +91,7 @@ "preversion": "npm run build && git add dist", "pretest": "npm run build", "test": "mocha 'test/**/*.{coffee,ts}'", - "test:legacy": "mocha --loader=./test/loaders/legacy/all.mjs 'test/**/*.{coffee,ts}'" + "test:legacy": "mocha --loader=./test/loaders/legacy/all.js 'test/**/*.{coffee,ts}'" }, "type": "module", "types": "dist/esm/index.d.ts", diff --git a/packages/csv/test/loaders/all.js b/packages/csv/test/loaders/all.js new file mode 100644 index 000000000..344154640 --- /dev/null +++ b/packages/csv/test/loaders/all.js @@ -0,0 +1,16 @@ + +import * as coffee from './coffee.js' +import * as ts from 'ts-node/esm' + +const coffeeRegex = /\.coffee$|\.litcoffee$|\.coffee\.md$/; +const tsRegex = /\.ts$/; + +export function load(url, context, next) { + if (coffeeRegex.test(url)) { + return coffee.load.apply(this, arguments) + } + if (tsRegex.test(url)) { + return ts.load.apply(this, arguments) + } + return next(url, context, next); +} diff --git a/packages/csv/test/loaders/all.mjs b/packages/csv/test/loaders/all.mjs deleted file mode 100644 index affe659fd..000000000 --- a/packages/csv/test/loaders/all.mjs +++ /dev/null @@ -1,26 +0,0 @@ - -import * as coffee from './coffee.mjs' -import * as ts from 'ts-node/esm' - -const coffeeRegex = /\.coffee$|\.litcoffee$|\.coffee\.md$/; -const tsRegex = /\.ts$/; - -export async function resolve(specifier, context, defaultResolve) { - if (coffeeRegex.test(specifier)) { - return coffee.resolve.apply(this, arguments) - } - if (tsRegex.test(specifier)) { - return ts.resolve.apply(this, arguments) - } - return defaultResolve(specifier, context, defaultResolve); -} - -export function load(url, context, defaultLoad) { - if (coffeeRegex.test(url)) { - return coffee.load.apply(this, arguments) - } - if (tsRegex.test(url)) { - return ts.load.apply(this, arguments) - } - return defaultLoad(url, context, defaultLoad); -} diff --git a/packages/csv/test/loaders/coffee.js b/packages/csv/test/loaders/coffee.js new file mode 100644 index 000000000..1252cdb81 --- /dev/null +++ b/packages/csv/test/loaders/coffee.js @@ -0,0 +1,13 @@ +import CoffeeScript from 'coffeescript'; + +const coffeeRegex = /\.coffee$|\.litcoffee$|\.coffee\.md$/; + +export async function load(url, context, next) { + if (coffeeRegex.test(url)) { + const format = 'module'; + const { source: rawSource } = await next(url, { format }); + const source = CoffeeScript.compile(rawSource.toString(), { bare: true }); + return {format, source}; + } + return next(url, context); +} diff --git a/packages/csv/test/loaders/coffee.mjs b/packages/csv/test/loaders/coffee.mjs deleted file mode 100644 index f54674cf6..000000000 --- a/packages/csv/test/loaders/coffee.mjs +++ /dev/null @@ -1,36 +0,0 @@ -// coffeescript-loader.mjs -import { URL, pathToFileURL } from 'url'; -import CoffeeScript from 'coffeescript'; -import { cwd } from 'process'; - -const baseURL = pathToFileURL(`${cwd()}/`).href; - -// CoffeeScript files end in .coffee, .litcoffee or .coffee.md. -const extensionsRegex = /\.coffee$|\.litcoffee$|\.coffee\.md$/; - -export function resolve(specifier, context, defaultResolve) { - const { parentURL = baseURL } = context; - // Node.js normally errors on unknown file extensions, so return a URL for - // specifiers ending in the CoffeeScript file extensions. - if (extensionsRegex.test(specifier)) { - return { - url: new URL(specifier, parentURL).href - }; - } - return defaultResolve(specifier, context, defaultResolve); -} - -export async function load(url, context, defaultLoad) { - if (extensionsRegex.test(url)) { - const format = 'module' - const { source: rawSource } = await defaultLoad(url, { format }); - return { - format: 'module', - source: CoffeeScript.compile(rawSource.toString(), { - bare: true, - filename: url, - }) - }; - } - return defaultLoad(url, context, defaultLoad); -} diff --git a/packages/csv/test/loaders/legacy/all.js b/packages/csv/test/loaders/legacy/all.js new file mode 100644 index 000000000..c95db0b56 --- /dev/null +++ b/packages/csv/test/loaders/legacy/all.js @@ -0,0 +1,37 @@ + +import * as coffee from './coffee.js' +import * as ts from 'ts-node/esm' + +const coffeeRegex = /\.coffee$|\.litcoffee$|\.coffee\.md$/; +const tsRegex = /\.ts$/; + +export function resolve(specifier, context, defaultResolve) { + if (coffeeRegex.test(specifier)) { + return coffee.resolve.apply(this, arguments) + } + if (tsRegex.test(specifier)) { + return ts.resolve.apply(this, arguments) + } + return ts.resolve.apply(this, arguments); +} + +export function getFormat(url, context, defaultGetFormat) { + if (coffeeRegex.test(url)) { + return coffee.getFormat.apply(this, arguments) + } + if (tsRegex.test(url)) { + return ts.getFormat.apply(this, arguments) + } + return ts.getFormat.apply(this, arguments); +} + +export function transformSource(source, context, defaultTransformSource) { + const { url } = context; + if (coffeeRegex.test(url)) { + return coffee.transformSource.apply(this, arguments) + } + if (tsRegex.test(url)) { + return ts.transformSource.apply(this, arguments) + } + return ts.transformSource.apply(this, arguments); +} diff --git a/packages/csv/test/loaders/legacy/all.mjs b/packages/csv/test/loaders/legacy/all.mjs deleted file mode 100644 index 4a3828589..000000000 --- a/packages/csv/test/loaders/legacy/all.mjs +++ /dev/null @@ -1,37 +0,0 @@ - -import * as coffee from './coffee.mjs' -import * as ts from 'ts-node/esm' - -const coffeeRegex = /\.coffee$|\.litcoffee$|\.coffee\.md$/; -const tsRegex = /\.ts$/; - -export function resolve(specifier, context, defaultResolve) { - if (coffeeRegex.test(specifier)) { - return coffee.resolve.apply(this, arguments) - } - if (tsRegex.test(specifier)) { - return ts.resolve.apply(this, arguments) - } - return ts.resolve.apply(this, arguments); -} - -export function getFormat(url, context, defaultGetFormat) { - if (coffeeRegex.test(url)) { - return coffee.getFormat.apply(this, arguments) - } - if (tsRegex.test(url)) { - return ts.getFormat.apply(this, arguments) - } - return ts.getFormat.apply(this, arguments); -} - -export function transformSource(source, context, defaultTransformSource) { - const { url } = context; - if (coffeeRegex.test(url)) { - return coffee.transformSource.apply(this, arguments) - } - if (tsRegex.test(url)) { - return ts.transformSource.apply(this, arguments) - } - return ts.transformSource.apply(this, arguments); -} diff --git a/packages/csv/test/loaders/legacy/coffee.mjs b/packages/csv/test/loaders/legacy/coffee.js similarity index 100% rename from packages/csv/test/loaders/legacy/coffee.mjs rename to packages/csv/test/loaders/legacy/coffee.js diff --git a/packages/stream-transform/package.json b/packages/stream-transform/package.json index af19ee657..20d4d77c2 100644 --- a/packages/stream-transform/package.json +++ b/packages/stream-transform/package.json @@ -48,7 +48,7 @@ "main": "./dist/cjs/index.cjs", "mocha": { "inline-diffs": true, - "loader": "./test/loaders/all.mjs", + "loader": "./test/loaders/all.js", "recursive": true, "reporter": "spec", "require": [ @@ -73,7 +73,7 @@ "preversion": "npm run build && git add dist", "pretest": "npm run build", "test": "mocha 'test/**/*.{coffee,ts}'", - "test:legacy": "mocha --loader=./test/loaders/legacy/all.mjs 'test/**/*.{coffee,ts}'" + "test:legacy": "mocha --loader=./test/loaders/legacy/all.js 'test/**/*.{coffee,ts}'" }, "type": "module", "types": "dist/esm/index.d.ts", diff --git a/packages/stream-transform/test/loaders/all.js b/packages/stream-transform/test/loaders/all.js new file mode 100644 index 000000000..344154640 --- /dev/null +++ b/packages/stream-transform/test/loaders/all.js @@ -0,0 +1,16 @@ + +import * as coffee from './coffee.js' +import * as ts from 'ts-node/esm' + +const coffeeRegex = /\.coffee$|\.litcoffee$|\.coffee\.md$/; +const tsRegex = /\.ts$/; + +export function load(url, context, next) { + if (coffeeRegex.test(url)) { + return coffee.load.apply(this, arguments) + } + if (tsRegex.test(url)) { + return ts.load.apply(this, arguments) + } + return next(url, context, next); +} diff --git a/packages/stream-transform/test/loaders/all.mjs b/packages/stream-transform/test/loaders/all.mjs deleted file mode 100644 index affe659fd..000000000 --- a/packages/stream-transform/test/loaders/all.mjs +++ /dev/null @@ -1,26 +0,0 @@ - -import * as coffee from './coffee.mjs' -import * as ts from 'ts-node/esm' - -const coffeeRegex = /\.coffee$|\.litcoffee$|\.coffee\.md$/; -const tsRegex = /\.ts$/; - -export async function resolve(specifier, context, defaultResolve) { - if (coffeeRegex.test(specifier)) { - return coffee.resolve.apply(this, arguments) - } - if (tsRegex.test(specifier)) { - return ts.resolve.apply(this, arguments) - } - return defaultResolve(specifier, context, defaultResolve); -} - -export function load(url, context, defaultLoad) { - if (coffeeRegex.test(url)) { - return coffee.load.apply(this, arguments) - } - if (tsRegex.test(url)) { - return ts.load.apply(this, arguments) - } - return defaultLoad(url, context, defaultLoad); -} diff --git a/packages/stream-transform/test/loaders/coffee.js b/packages/stream-transform/test/loaders/coffee.js new file mode 100644 index 000000000..1252cdb81 --- /dev/null +++ b/packages/stream-transform/test/loaders/coffee.js @@ -0,0 +1,13 @@ +import CoffeeScript from 'coffeescript'; + +const coffeeRegex = /\.coffee$|\.litcoffee$|\.coffee\.md$/; + +export async function load(url, context, next) { + if (coffeeRegex.test(url)) { + const format = 'module'; + const { source: rawSource } = await next(url, { format }); + const source = CoffeeScript.compile(rawSource.toString(), { bare: true }); + return {format, source}; + } + return next(url, context); +} diff --git a/packages/stream-transform/test/loaders/coffee.mjs b/packages/stream-transform/test/loaders/coffee.mjs deleted file mode 100644 index f54674cf6..000000000 --- a/packages/stream-transform/test/loaders/coffee.mjs +++ /dev/null @@ -1,36 +0,0 @@ -// coffeescript-loader.mjs -import { URL, pathToFileURL } from 'url'; -import CoffeeScript from 'coffeescript'; -import { cwd } from 'process'; - -const baseURL = pathToFileURL(`${cwd()}/`).href; - -// CoffeeScript files end in .coffee, .litcoffee or .coffee.md. -const extensionsRegex = /\.coffee$|\.litcoffee$|\.coffee\.md$/; - -export function resolve(specifier, context, defaultResolve) { - const { parentURL = baseURL } = context; - // Node.js normally errors on unknown file extensions, so return a URL for - // specifiers ending in the CoffeeScript file extensions. - if (extensionsRegex.test(specifier)) { - return { - url: new URL(specifier, parentURL).href - }; - } - return defaultResolve(specifier, context, defaultResolve); -} - -export async function load(url, context, defaultLoad) { - if (extensionsRegex.test(url)) { - const format = 'module' - const { source: rawSource } = await defaultLoad(url, { format }); - return { - format: 'module', - source: CoffeeScript.compile(rawSource.toString(), { - bare: true, - filename: url, - }) - }; - } - return defaultLoad(url, context, defaultLoad); -} diff --git a/packages/stream-transform/test/loaders/legacy/all.js b/packages/stream-transform/test/loaders/legacy/all.js new file mode 100644 index 000000000..c95db0b56 --- /dev/null +++ b/packages/stream-transform/test/loaders/legacy/all.js @@ -0,0 +1,37 @@ + +import * as coffee from './coffee.js' +import * as ts from 'ts-node/esm' + +const coffeeRegex = /\.coffee$|\.litcoffee$|\.coffee\.md$/; +const tsRegex = /\.ts$/; + +export function resolve(specifier, context, defaultResolve) { + if (coffeeRegex.test(specifier)) { + return coffee.resolve.apply(this, arguments) + } + if (tsRegex.test(specifier)) { + return ts.resolve.apply(this, arguments) + } + return ts.resolve.apply(this, arguments); +} + +export function getFormat(url, context, defaultGetFormat) { + if (coffeeRegex.test(url)) { + return coffee.getFormat.apply(this, arguments) + } + if (tsRegex.test(url)) { + return ts.getFormat.apply(this, arguments) + } + return ts.getFormat.apply(this, arguments); +} + +export function transformSource(source, context, defaultTransformSource) { + const { url } = context; + if (coffeeRegex.test(url)) { + return coffee.transformSource.apply(this, arguments) + } + if (tsRegex.test(url)) { + return ts.transformSource.apply(this, arguments) + } + return ts.transformSource.apply(this, arguments); +} diff --git a/packages/stream-transform/test/loaders/legacy/all.mjs b/packages/stream-transform/test/loaders/legacy/all.mjs deleted file mode 100644 index 4a3828589..000000000 --- a/packages/stream-transform/test/loaders/legacy/all.mjs +++ /dev/null @@ -1,37 +0,0 @@ - -import * as coffee from './coffee.mjs' -import * as ts from 'ts-node/esm' - -const coffeeRegex = /\.coffee$|\.litcoffee$|\.coffee\.md$/; -const tsRegex = /\.ts$/; - -export function resolve(specifier, context, defaultResolve) { - if (coffeeRegex.test(specifier)) { - return coffee.resolve.apply(this, arguments) - } - if (tsRegex.test(specifier)) { - return ts.resolve.apply(this, arguments) - } - return ts.resolve.apply(this, arguments); -} - -export function getFormat(url, context, defaultGetFormat) { - if (coffeeRegex.test(url)) { - return coffee.getFormat.apply(this, arguments) - } - if (tsRegex.test(url)) { - return ts.getFormat.apply(this, arguments) - } - return ts.getFormat.apply(this, arguments); -} - -export function transformSource(source, context, defaultTransformSource) { - const { url } = context; - if (coffeeRegex.test(url)) { - return coffee.transformSource.apply(this, arguments) - } - if (tsRegex.test(url)) { - return ts.transformSource.apply(this, arguments) - } - return ts.transformSource.apply(this, arguments); -} diff --git a/packages/stream-transform/test/loaders/legacy/coffee.mjs b/packages/stream-transform/test/loaders/legacy/coffee.js similarity index 100% rename from packages/stream-transform/test/loaders/legacy/coffee.mjs rename to packages/stream-transform/test/loaders/legacy/coffee.js From 4958529c229ff0188a187c15e76e32f0adfe43c3 Mon Sep 17 00:00:00 2001 From: David Worms Date: Mon, 21 Feb 2022 16:21:27 +0100 Subject: [PATCH 04/19] test: coffee esm loader options --- demo/esm/test/loaders/coffee.js | 13 ++++++++++--- demo/issues-esm/test/loaders/coffee.js | 13 ++++++++++--- packages/csv-generate/test/loaders/coffee.js | 13 ++++++++++--- packages/csv-parse/test/loaders/coffee.js | 13 ++++++++++--- packages/csv-stringify/test/loaders/coffee.js | 13 ++++++++++--- packages/csv/test/loaders/coffee.js | 13 ++++++++++--- packages/stream-transform/test/loaders/coffee.js | 13 ++++++++++--- 7 files changed, 70 insertions(+), 21 deletions(-) diff --git a/demo/esm/test/loaders/coffee.js b/demo/esm/test/loaders/coffee.js index 1252cdb81..c7b277cb7 100644 --- a/demo/esm/test/loaders/coffee.js +++ b/demo/esm/test/loaders/coffee.js @@ -1,12 +1,19 @@ import CoffeeScript from 'coffeescript'; -const coffeeRegex = /\.coffee$|\.litcoffee$|\.coffee\.md$/; +// See https://github.com/nodejs/node/issues/36396 +const extensionsRegex = /\.coffee$|\.litcoffee$|\.coffee\.md$/; export async function load(url, context, next) { - if (coffeeRegex.test(url)) { + if (extensionsRegex.test(url)) { const format = 'module'; const { source: rawSource } = await next(url, { format }); - const source = CoffeeScript.compile(rawSource.toString(), { bare: true }); + const source = CoffeeScript.compile(rawSource.toString(), { + bare: true, + inlineMap: true, + filename: url, + header: false, + sourceMap: false, + }); return {format, source}; } return next(url, context); diff --git a/demo/issues-esm/test/loaders/coffee.js b/demo/issues-esm/test/loaders/coffee.js index 1252cdb81..c7b277cb7 100644 --- a/demo/issues-esm/test/loaders/coffee.js +++ b/demo/issues-esm/test/loaders/coffee.js @@ -1,12 +1,19 @@ import CoffeeScript from 'coffeescript'; -const coffeeRegex = /\.coffee$|\.litcoffee$|\.coffee\.md$/; +// See https://github.com/nodejs/node/issues/36396 +const extensionsRegex = /\.coffee$|\.litcoffee$|\.coffee\.md$/; export async function load(url, context, next) { - if (coffeeRegex.test(url)) { + if (extensionsRegex.test(url)) { const format = 'module'; const { source: rawSource } = await next(url, { format }); - const source = CoffeeScript.compile(rawSource.toString(), { bare: true }); + const source = CoffeeScript.compile(rawSource.toString(), { + bare: true, + inlineMap: true, + filename: url, + header: false, + sourceMap: false, + }); return {format, source}; } return next(url, context); diff --git a/packages/csv-generate/test/loaders/coffee.js b/packages/csv-generate/test/loaders/coffee.js index 1252cdb81..c7b277cb7 100644 --- a/packages/csv-generate/test/loaders/coffee.js +++ b/packages/csv-generate/test/loaders/coffee.js @@ -1,12 +1,19 @@ import CoffeeScript from 'coffeescript'; -const coffeeRegex = /\.coffee$|\.litcoffee$|\.coffee\.md$/; +// See https://github.com/nodejs/node/issues/36396 +const extensionsRegex = /\.coffee$|\.litcoffee$|\.coffee\.md$/; export async function load(url, context, next) { - if (coffeeRegex.test(url)) { + if (extensionsRegex.test(url)) { const format = 'module'; const { source: rawSource } = await next(url, { format }); - const source = CoffeeScript.compile(rawSource.toString(), { bare: true }); + const source = CoffeeScript.compile(rawSource.toString(), { + bare: true, + inlineMap: true, + filename: url, + header: false, + sourceMap: false, + }); return {format, source}; } return next(url, context); diff --git a/packages/csv-parse/test/loaders/coffee.js b/packages/csv-parse/test/loaders/coffee.js index 1252cdb81..c7b277cb7 100644 --- a/packages/csv-parse/test/loaders/coffee.js +++ b/packages/csv-parse/test/loaders/coffee.js @@ -1,12 +1,19 @@ import CoffeeScript from 'coffeescript'; -const coffeeRegex = /\.coffee$|\.litcoffee$|\.coffee\.md$/; +// See https://github.com/nodejs/node/issues/36396 +const extensionsRegex = /\.coffee$|\.litcoffee$|\.coffee\.md$/; export async function load(url, context, next) { - if (coffeeRegex.test(url)) { + if (extensionsRegex.test(url)) { const format = 'module'; const { source: rawSource } = await next(url, { format }); - const source = CoffeeScript.compile(rawSource.toString(), { bare: true }); + const source = CoffeeScript.compile(rawSource.toString(), { + bare: true, + inlineMap: true, + filename: url, + header: false, + sourceMap: false, + }); return {format, source}; } return next(url, context); diff --git a/packages/csv-stringify/test/loaders/coffee.js b/packages/csv-stringify/test/loaders/coffee.js index 1252cdb81..c7b277cb7 100644 --- a/packages/csv-stringify/test/loaders/coffee.js +++ b/packages/csv-stringify/test/loaders/coffee.js @@ -1,12 +1,19 @@ import CoffeeScript from 'coffeescript'; -const coffeeRegex = /\.coffee$|\.litcoffee$|\.coffee\.md$/; +// See https://github.com/nodejs/node/issues/36396 +const extensionsRegex = /\.coffee$|\.litcoffee$|\.coffee\.md$/; export async function load(url, context, next) { - if (coffeeRegex.test(url)) { + if (extensionsRegex.test(url)) { const format = 'module'; const { source: rawSource } = await next(url, { format }); - const source = CoffeeScript.compile(rawSource.toString(), { bare: true }); + const source = CoffeeScript.compile(rawSource.toString(), { + bare: true, + inlineMap: true, + filename: url, + header: false, + sourceMap: false, + }); return {format, source}; } return next(url, context); diff --git a/packages/csv/test/loaders/coffee.js b/packages/csv/test/loaders/coffee.js index 1252cdb81..c7b277cb7 100644 --- a/packages/csv/test/loaders/coffee.js +++ b/packages/csv/test/loaders/coffee.js @@ -1,12 +1,19 @@ import CoffeeScript from 'coffeescript'; -const coffeeRegex = /\.coffee$|\.litcoffee$|\.coffee\.md$/; +// See https://github.com/nodejs/node/issues/36396 +const extensionsRegex = /\.coffee$|\.litcoffee$|\.coffee\.md$/; export async function load(url, context, next) { - if (coffeeRegex.test(url)) { + if (extensionsRegex.test(url)) { const format = 'module'; const { source: rawSource } = await next(url, { format }); - const source = CoffeeScript.compile(rawSource.toString(), { bare: true }); + const source = CoffeeScript.compile(rawSource.toString(), { + bare: true, + inlineMap: true, + filename: url, + header: false, + sourceMap: false, + }); return {format, source}; } return next(url, context); diff --git a/packages/stream-transform/test/loaders/coffee.js b/packages/stream-transform/test/loaders/coffee.js index 1252cdb81..c7b277cb7 100644 --- a/packages/stream-transform/test/loaders/coffee.js +++ b/packages/stream-transform/test/loaders/coffee.js @@ -1,12 +1,19 @@ import CoffeeScript from 'coffeescript'; -const coffeeRegex = /\.coffee$|\.litcoffee$|\.coffee\.md$/; +// See https://github.com/nodejs/node/issues/36396 +const extensionsRegex = /\.coffee$|\.litcoffee$|\.coffee\.md$/; export async function load(url, context, next) { - if (coffeeRegex.test(url)) { + if (extensionsRegex.test(url)) { const format = 'module'; const { source: rawSource } = await next(url, { format }); - const source = CoffeeScript.compile(rawSource.toString(), { bare: true }); + const source = CoffeeScript.compile(rawSource.toString(), { + bare: true, + inlineMap: true, + filename: url, + header: false, + sourceMap: false, + }); return {format, source}; } return next(url, context); From bb340a909776564b149dbdac8c03889b9ce89158 Mon Sep 17 00:00:00 2001 From: David Worms Date: Wed, 2 Mar 2022 23:07:00 +0100 Subject: [PATCH 05/19] docs(csv-stringify): bom sample --- packages/csv-stringify/samples/option.bom.js | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 packages/csv-stringify/samples/option.bom.js diff --git a/packages/csv-stringify/samples/option.bom.js b/packages/csv-stringify/samples/option.bom.js new file mode 100644 index 000000000..771e01369 --- /dev/null +++ b/packages/csv-stringify/samples/option.bom.js @@ -0,0 +1,10 @@ + +import assert from 'assert'; +import { stringify } from 'csv-stringify/sync'; + +const data = stringify([ + [ 'a', 'b', 'c' ] +], { + bom: true +}); +assert.deepStrictEqual(data, "\ufeffa,b,c\n"); From 3d85a411007416f3cb750ca6b427f55c0331a8b8 Mon Sep 17 00:00:00 2001 From: David Worms Date: Wed, 23 Mar 2022 09:37:47 +0100 Subject: [PATCH 06/19] feat(csv-issues-cjs): 330 sample code --- demo/issues-cjs/lib/330.1.js | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 demo/issues-cjs/lib/330.1.js diff --git a/demo/issues-cjs/lib/330.1.js b/demo/issues-cjs/lib/330.1.js new file mode 100644 index 000000000..f1dcd4189 --- /dev/null +++ b/demo/issues-cjs/lib/330.1.js @@ -0,0 +1,13 @@ +const {parse} = require('csv/sync'); + +const input = ` +Date, Type, Description, Value, Balance, Account Name, Account Number + +11/02/2022,BAC,"'FOO",10.00,33432.80,"'REF","'123123-12312312", +`; + +const output = parse(input, { + skip_empty_lines: true, + relax_column_count: true +}); +console.log(output); From 64afead8dc41b9d379c9761ddb70d6a29251b4e2 Mon Sep 17 00:00:00 2001 From: David Worms Date: Wed, 23 Mar 2022 09:41:34 +0100 Subject: [PATCH 07/19] fix(csv-demo-esm): csv dependencies --- demo/esm/package.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/demo/esm/package.json b/demo/esm/package.json index b24abf35a..a74891283 100644 --- a/demo/esm/package.json +++ b/demo/esm/package.json @@ -5,6 +5,10 @@ "license": "MIT", "type": "module", "private": true, + "dependencies": { + "csv": "^6.0.5", + "csv-parse": "^5.0.4" + }, "devDependencies": { "coffeescript": "^2.6.1", "mocha": "^9.2.0", From f79fe219577b3516d662df62f8802f8714c4abdb Mon Sep 17 00:00:00 2001 From: David Worms Date: Mon, 4 Apr 2022 09:17:12 +0200 Subject: [PATCH 08/19] docs(csv-issues-cjs): 265 sample --- demo/issues-cjs/lib/265.1.js | 14 + demo/issues-cjs/lib/265.tubs.csv | 496 +++++++++++++++++++++++++++++++ 2 files changed, 510 insertions(+) create mode 100644 demo/issues-cjs/lib/265.1.js create mode 100644 demo/issues-cjs/lib/265.tubs.csv diff --git a/demo/issues-cjs/lib/265.1.js b/demo/issues-cjs/lib/265.1.js new file mode 100644 index 000000000..5a7dcace2 --- /dev/null +++ b/demo/issues-cjs/lib/265.1.js @@ -0,0 +1,14 @@ + +const {parse} = require('csv-parse'); +const fs = require('fs'); + +fs.createReadStream(`${__dirname}/265.tubs.csv`) +.pipe(parse({ + delimiter: ',', + columns: false, + skip_empty_lines: true, + trim: true, +})) +.on('data', data => { + console.log(data); +}); diff --git a/demo/issues-cjs/lib/265.tubs.csv b/demo/issues-cjs/lib/265.tubs.csv new file mode 100644 index 000000000..a60f220c4 --- /dev/null +++ b/demo/issues-cjs/lib/265.tubs.csv @@ -0,0 +1,496 @@ +-- BATHTUBS --,Description,Finish,Model #,List Price,Vendor,Mult.,Specs,Images,ImageFormula,RETURN TO TABLE OF CONTENTS +Bain Ultra Charism 66x36x24 (Acrylic / Freestanding Oval / Center),"Charism acrylic freestanding oval tub | Nominal Size: 64"" x 36"" x 24"" | Bathing Well: 41 3/4"" x 23 5/8"" x 16"" (51 gallon capacity); Includes Geysair® Hydro-Thermo Massage, wall-mount control, and heated backrest; Requires integrated waste & overflow (BOVERINI-XX) and template (90000067)",White,BCH3OF00T-01,"$8,395.00",Robinson,0.65,https://www.bainultra.com/bath-collections/charism-collection/charism-6436,,"=IMAGE(""https://bainultra.com/sites/default/files/2018-01/Charism-6436.jpg"",4,175,175)", +Bain Ultra Esthesia 66x36x25 (Acrylic / Freestanding Oval / Center),"Esthesia acrylic freestanding oval tub | Nominal Size: 64"" x 36"" x 25"" | Bathing Well: 47 3/8"" x 23"" x 15 7/8"" (59 gallon capacity); Includes Geysair® Hydro-Thermo Massage, wall-mount control, and heated backrest; Requires integrated waste & overflow (BOVERINI-XX) and template (90000066)",White,BEA3RF00T-01,"$7,125.00",Robinson,0.65,https://www.bainultra.com/bath-collections/esthesia-collection/esthesia-6436,,"=IMAGE(""https://bainultra.com/sites/default/files/2018-01/Esthesia-6436-freestanding.jpg"",4,150,150)", +"Bain Ultra Sanos 66x36x24, Flat Deck (Acrylic / Freestanding Oval / Center)","Sanos acrylic freestanding oval tub with flat deck | Nominal Size: 66"" x 36"" x 24"" | Bathing Well: 42 3/8"" x 22 1/4"" x 14 7/8"" (48 gallon capacity); Includes Geysair® Hydro-Thermo Massage, wall-mount control, and heated backrest; Requires separate waste & overflow (BOVERD13-XX)",White,BBSLOFN0T-01,"$7,175.00",Robinson,0.65,https://www.bainultra.com/bath-collections/balneo-collection/sanos-6636#show,,"=IMAGE(""https://bainultra.com/sites/default/files/2018-01/Balneo-sanos-6636-flat-deck.jpg"",4,175,175)", +"Bain Ultra Sanos 66x36x24, Rolled Deck (Acrylic / Freestanding Oval / Center)","Sanos acrylic freestanding oval tub with flat deck | Nominal Size: 66"" x 36"" x 24"" | Bathing Well: 42 3/8"" x 22 1/4"" x 14 7/8"" (48 gallon capacity); Includes Geysair® Hydro-Thermo Massage, wall-mount control, and heated backrest; Requires separate waste & overflow (BOVERD13-XX)",White,BBSLOFP0T-01,"$7,175.00",Robinson,0.65,https://www.bainultra.com/bath-collections/balneo-collection/sanos-6636#show,,"=IMAGE(""https://bainultra.com/sites/default/files/2018-01/Balneo-sanos-6636-flat-deck.jpg"",4,175,175)", +Bain Ultra Scala 66x38x25 (Acrylic / Freestanding Oval / Center / BLK Feet),"Scala acrylic freestanding oval tub with MATTE BLACK feet | Nominal Size: 66"" x 38"" x 25"" | Bathing Well: 44 1/2"" x 23 1/8"" x 17 1/2"" (62 gallon capacity); Includes Geysair® Hydro-Thermo Massage, wall-mount control, and heated backrest; Requires integrated waste & overflow (BOVERINI-XX) and template (90000077)",White with Matte Black Feet,"BSCMOF00T-01 (Tub), BFEETSC-19 (Feet, Specify Finish)",#N/A,Robinson,0.65,https://www.bainultra.com/bath-collections/scala-collection/scala-6638,,"=IMAGE(""https://bainultra.com/sites/default/files/2018-12/scal_6638_freestanding_low_0.jpg"",4,175,175)", +Bain Ultra Scala 66x38x25 (Acrylic / Freestanding Oval / Center / BSN Feet),"Scala acrylic freestanding oval tub with BRUSHED NICKEL feet | Nominal Size: 66"" x 38"" x 25"" | Bathing Well: 44 1/2"" x 23 1/8"" x 17 1/2"" (62 gallon capacity); Includes Geysair® Hydro-Thermo Massage, wall-mount control, and heated backrest; Requires integrated waste & overflow (BOVERINI-XX) and template (90000077)",White with Brushed Nickel Feet,"BSCMOF00T-01 (Tub), BFEETSC-09 (Feet, Specify Finish)","$7,450.00",Robinson,0.65,https://www.bainultra.com/bath-collections/scala-collection/scala-6638,,"=IMAGE(""https://bainultra.com/sites/default/files/2018-12/scal_6638_freestanding_low_0.jpg"",4,175,175)", +Bain Ultra Scala 66x38x25 (Acrylic / Freestanding Oval / Center / CHR Feet),"Scala acrylic freestanding oval tub with CHROME feet | Nominal Size: 66"" x 38"" x 25"" | Bathing Well: 44 1/2"" x 23 1/8"" x 17 1/2"" (62 gallon capacity); Includes Geysair® Hydro-Thermo Massage, wall-mount control, and heated backrest; Requires integrated waste & overflow (BOVERINI-XX) and template (90000077)",White with Chrome Feet,"BSCMOF00T-01 (Tub), BFEETSC-07 (Feet, Specify Finish)","$7,450.00",Robinson,0.65,https://www.bainultra.com/bath-collections/scala-collection/scala-6638,,"=IMAGE(""https://bainultra.com/sites/default/files/2018-12/scal_6638_freestanding_low_0.jpg"",4,175,175)", +Bain Ultra Scala 66x38x25 (Acrylic / Freestanding Oval / Center / GOLD Feet),"Scala acrylic freestanding oval tub with SATIN BRASS feet | Nominal Size: 66"" x 38"" x 25"" | Bathing Well: 44 1/2"" x 23 1/8"" x 17 1/2"" (62 gallon capacity); Includes Geysair® Hydro-Thermo Massage, wall-mount control, and heated backrest; Requires integrated waste & overflow (BOVERINI-XX) and template (90000077)",White with Satin Brass Feet,"BSCMOF00T-01 (Tub), BFEETSC-18 (Feet, Specify Finish)",#N/A,Robinson,0.65,https://www.bainultra.com/bath-collections/scala-collection/scala-6638,,"=IMAGE(""https://bainultra.com/sites/default/files/2018-12/scal_6638_freestanding_low_0.jpg"",4,175,175)", +Bain Ultra Scala 66x38x25 (Acrylic / Freestanding Oval / Center / PN Feet),"Scala acrylic freestanding oval tub with POLISHED NICKEL feet | Nominal Size: 66"" x 38"" x 25"" | Bathing Well: 44 1/2"" x 23 1/8"" x 17 1/2"" (62 gallon capacity); Includes Geysair® Hydro-Thermo Massage, wall-mount control, and heated backrest; Requires integrated waste & overflow (BOVERINI-XX) and template (90000077)",White with Polished Nickel Feet,"BSCMOF00T-01 (Tub), BFEETSC-10 (Feet, Specify Finish)","$7,450.00",Robinson,0.65,https://www.bainultra.com/bath-collections/scala-collection/scala-6638,,"=IMAGE(""https://bainultra.com/sites/default/files/2018-12/scal_6638_freestanding_low_0.jpg"",4,175,175)", +Bain Ultra Origami 72x36x22 ( Drop In / Alcove / Undermount / Center),,White,BOOVR120N-01,"$1,776.00",Granite Group,,https://bainultra.com/bath-collections/origami-collection/origami-7236-original-series,,"=IMAGE(""https://images.webfronts.com/cache/frosnckawxjq.jpg?imgeng=/w_300"",1)", +Fleurco Alto Petite 58x31x22 (Acrylic / Freestanding Oval / Center),"Alto Petite acrylic freestanding oval tub with deck space for faucet installation +- Nominal Size: 58"" x 31"" x 22"" +- Bathing Well: 38 3/8"" x 18 7/8"" x 14 3/4"" +- 62 gallon capacity +- Requires separate drain (DRAINRC-T-PVC)",White,BAL5831-18,"$3,108.00",Boston Basins,0.55,https://fleurco.com/bathtubs/alto-petite,,"=IMAGE(""https://fleurco.com/cms/controller/services/Thumbnail.ashx?width=600&fileName=/media/website_bathtubs.photo/aria_alto_petite.jpg"",1)", +Fleurco Concerto Grande 66x31x30 (Acrylic / Freestanding Oval),"Concerto Grande acrylic freestanding oval tub with deck space for faucet installation +- Nominal Size: 66 7/8"" x 31 1/2"" x 30 5/8"" +- Bathing Well: 45 3/4"" x 21 7/8"" x 14"" +- 63 gallon capacity +- Requires separate drain (DRAINRC-T-PVC)",White,BZCO6731-18,"$2,624.00",Boston Basins,0.55,https://fleurco.com/bathtubs/concerto-grande,,"=IMAGE(""https://fleurco.com/cms/controller/services/Thumbnail.ashx?width=200&fileName=/media/website_bathtubs.photo/opus_concerto_bzco6731.jpg"",1)", +Fleurco Libretto Petite 59x31x22 (Acrylic / Freestanding Oval / Center),"Libretto Petite acrylic freestanding oval tub with deck space for faucet installation | Nominal Size: 59"" x 31"" x 22"" | Bathing Well: 39 3/8"" x 18 7/8"" x 14 1/8"" (54 gallon capacity); Requires separate drain (DRAINRC-T-PVC)",White,BZLI5931-18,"$2,282.00",Boston Basins,0.55,https://fleurco.com/bathtubs/libretto-petite,,"=IMAGE(""https://fleurco.com/cms/controller/services/Thumbnail.ashx?width=600&fileName=/media/website_bathtubs.photo/opus_libretto_5931-letter.jpg"",1)", +Fleurco Motif 60x32 (Acrylic / Alcove / LH),"Motif acrylic alcove bath with integral apron, integral flange +- Left-hand drain +- Nominal Size: 60"" x 32"" x 21"" +- Bathing Well: 48 1/2"" x 24 5/8"" x 13 5/8"" +- 69 gallon capacity +- Requires SlideOn™ drain (DRNSLIDE-X-PVC)",White,"BZMO6032L-18","$1,577.00",Boston Basins,0.55,https://fleurco.com/bathtubs/motif-6032,,"=IMAGE(""https://fleurco.com/cms/controller/services/Thumbnail.ashx?width=200&fileName=/media/website_bathtubs.photo/opus_motif.jpg"",1)", +Fleurco Motif 60x32 (Acrylic / Alcove / RH),"Motif acrylic alcove bath with integral apron, integral flange +- Right-hand drain +- Nominal Size: 60"" x 32"" x 21"" +- Bathing Well: 48 1/2"" x 24 5/8"" x 13 5/8"" +- 69 gallon capacity +- Requires SlideOn™ drain (DRNSLIDE-X-PVC)",White,"BZMO6032R-18 +","$1,577.00",Boston Basins,0.55,https://fleurco.com/bathtubs/motif-6032,,"=IMAGE(""https://fleurco.com/cms/controller/services/Thumbnail.ashx?width=200&fileName=/media/website_bathtubs.photo/opus_motif.jpg"",1)", +Fleurco Waltz Petite 59x31x18 (Acrylic / Freestanding Rectangular / Center),"Libretto Waltz acrylic freestanding rectangular tub with deck space for faucet installation | Nominal Size: 59"" x 31"" x 18"" | Bathing Well: 43 3/8"" x 18 3/4"" x 14 1/8"" (60 gallon capacity); Requires separate drain (DRAINRC-T-PVC)",White,BZWA5931-18,"$2,457.90",Boston Basins,0.55,https://fleurco.com/bathtubs/waltz-petite,,"=IMAGE(""https://fleurco.com/cms/controller/services/Thumbnail.ashx?width=600&fileName=/media/website_bathtubs.photo/opus_walts_5931_bleed-letter.jpg"",1)", +Kohler Archer 60x30x19 (Acrylic / Alcove / LH),"Archer acrylic alcove bath with integral apron, integral flange +- Left-hand drain +- Nominal Size: 60"" x 30"" x 19"" +- Bathing Well: 45 1/8"" x 19 11/16 x 15 1/8"" +- 57 gallon capacity) +- Requires ClearFlo drain (K-7272)",White,K-1946-LA-0,$949.35,Granite Group,0.6,https://www.us.kohler.com/us/archer-60-x-30-alcove-bath-w-integral-apron-integral-flange-and-left-hand-drain/productDetail/bathing/931163.htm?skuId=930924&brandId=1159437,,"=IMAGE(""https://kohler.scene7.com/is/image/PAWEB/Category_Template?$PDPcon$&$gradient_src=PAWEB%2Forganic-gradient&$shadow_src=PAWEB%2FBlank&$Badge1_src=PAWEB%2FBlank&$Badge4_src=PAWEB%2FBlank&$Badge3_src=PAWEB%2FBlank&$Badge2_src=PAWEB%2FBlank&$product_src=is{PAWEB%2Fzac32872_rgb}"",1)", +Kohler Archer 60x30x19 (Acrylic / Alcove / RH),"Archer acrylic alcove bath with integral apron, integral flange +- Right-hand drain +- Nominal Size: 60"" x 30"" x 19"" +- Bathing Well: 45 1/8"" x 19 11/16 x 15 1/8"" +- 57 gallon capacity) +- Requires ClearFlo drain (K-7272)",White,K-1946-RA-0,$949.35,Granite Group,0.6,https://www.us.kohler.com/us/archer-60-x-30-alcove-bath-w-integral-apron-integral-flange-and-right-hand-drain/productDetail/bathing/931164.htm,,"=IMAGE(""https://kohler.scene7.com/is/image/PAWEB/Category_Template?$PDPcon$&$gradient_src=PAWEB%2Forganic-gradient&$shadow_src=PAWEB%2FBlank&$Badge1_src=PAWEB%2FBlank&$Badge4_src=PAWEB%2FBlank&$Badge3_src=PAWEB%2FBlank&$Badge2_src=PAWEB%2FBlank&$product_src=is{PAWEB%2Fzac32874_rgb}"",1)", +"Kohler Archer 60x30x19, No Apron (Acrylic / Alcove / LH)","Archer acrylic alcove bath with integral flange and left-hand drain | Nominal Size: 60"" x 30"" x 19"" | Bathing Well: 45 1/8"" x 19 11/16 x 15 1/8"" (57 gallon capacity); Requires ClearFlo drain (K-7272)",White,K-1946-L-0,$804.50,Granite Group,0.6,https://www.us.kohler.com/us/archer-60-x-30-alcove-bath-with-integral-flange-and-left-hand-drain/productDetail/bathing/922327.htm,,"=IMAGE(""https://kohler.scene7.com/is/image/PAWEB/Category_Template?$PDPcon$&$gradient_src=PAWEB%2Forganic-gradient&$shadow_src=PAWEB%2FBlank&$Badge1_src=PAWEB%2FBlank&$Badge4_src=PAWEB%2FBlank&$Badge3_src=PAWEB%2FBlank&$Badge2_src=PAWEB%2FBlank&$product_src=is{PAWEB%2Fzab28743_rgb}"",1)", +"Kohler Archer 60x30x19, No Apron (Acrylic / Alcove / RH)","Archer acrylic alcove bath with integral flange, less apron +- Right-hand drain +- Nominal Size: 60"" x 30"" x 19"" +- Bathing Well: 45 1/8"" x 19 11/16 x 15 1/8"" +- 57 gallon capacity) +- Requires ClearFlo drain (K-7272)",White,K-1946-R-0,$828.80,Granite Group,0.6,https://www.us.kohler.com/us/archer-60-x-30-alcove-bath-with-integral-flange-and-right-hand-drain/productDetail/bathing/922328.htm,,"=IMAGE(""https://kohler.scene7.com/is/image/PAWEB/Category_Template?$PDPcon$&$gradient_src=PAWEB%2Forganic-gradient&$shadow_src=PAWEB%2FBlank&$Badge1_src=PAWEB%2FBlank&$Badge4_src=PAWEB%2FBlank&$Badge3_src=PAWEB%2FBlank&$Badge2_src=PAWEB%2FBlank&$product_src=is{PAWEB%2Fzab28747_rgb}"",1)", +Kohler Archer 60x32x19 (Acrylic / Alcove / RH),"Archer® acrylic alcove bath with integral apron, integral flange +- Right-hand drain +- Nominal Size: 60"" x 32"" x 19"" +- Bathing Well: 45 1/8"" x 19 11/16"" x 15"" +- 58 gallon capacity +- Requires ClearFlo drain (K-7272) +- Therapy options available",White,K-1123-RA-0,"$1,097.80",Granite Group,0.6,https://www.us.kohler.com/us/archer-60-x-32-alcove-bath-w-integral-apron-integral-flange-and-right-hand-drain/productDetail/bathing/425547.htm?skuId=394494&brandId=empty&,,"=IMAGE(""https://kohler.scene7.com/is/image/PAWEB/Category_Template?$PDPcon$&$gradient_src=PAWEB%2Forganic-gradient&$shadow_src=PAWEB%2FBlank&$Badge1_src=PAWEB%2FBlank&$Badge4_src=PAWEB%2FBlank&$Badge3_src=PAWEB%2FBlank&$Badge2_src=PAWEB%2FBlank&$product_src=is{PAWEB%2Fzac32866_rgb}"",1)", +Kohler Archer 60x32x19 (Acrylic / Alcove / LH),"Archer® acrylic alcove bath with integral apron, integral flange +- Left-hand drain +- Nominal Size: 60"" x 32"" x 19"" +- Bathing Well: 45 1/8"" x 19 11/16"" x 15"" +- 58 gallon capacity +- Requires ClearFlo drain (K-7272) +- Therapy options available",White,"K-1123-LA-0 +","$1,097.80",Granite Group,0.6,https://www.us.kohler.com/us/archer-60-x-32-alcove-bath-with-integral-apron-integral-flange-and-left-hand-drain/productDetail/bathing/425546.htm,,"=IMAGE(""https://kohler.scene7.com/is/image/PAWEB/Category_Template?$PDPcon$&$gradient_src=PAWEB%2Forganic-gradient&$shadow_src=PAWEB%2FBlank&$Badge1_src=PAWEB%2FBlank&$Badge4_src=PAWEB%2FBlank&$Badge3_src=PAWEB%2FBlank&$Badge2_src=PAWEB%2FBlank&$product_src=is{PAWEB%2Fzac32864_rgb}"",1)", +Kohler Archer 60x32x19 (Acrylic / Alcove / RH) (BISCUIT),"Archer® acrylic alcove bath with integral apron, integral flange +- Right-hand drain +- Nominal Size: 60"" x 32"" x 19"" +- Bathing Well: 45 1/8"" x 19 11/16"" x 15"" +- 58 gallon capacity +- Requires ClearFlo drain (K-7272) +- Therapy options available",Biscuit,K-1123-RA-96,"$1,097.80",Granite Group,0.6,https://www.us.kohler.com/us/archer-60-x-32-alcove-bath-w-integral-apron-integral-flange-and-right-hand-drain/productDetail/bathing/425547.htm?skuId=394501&brandId=empty&=undefined,,"=IMAGE(""https://kohler.scene7.com/is/image/PAWEB/Category_Template?$PDPcon$&$gradient_src=PAWEB%2Forganic-gradient&$shadow_src=PAWEB%2FBlank&$Badge1_src=PAWEB%2FBlank&$Badge4_src=PAWEB%2FBlank&$Badge3_src=PAWEB%2FBlank&$Badge2_src=PAWEB%2FBlank&$product_src=is{PAWEB%2Fzac32892_rgb}&fmt"",1)", +Kohler Archer 60x32x19 (Acrylic / Alcove / LH) (BISCUIT),"Archer® acrylic alcove bath with integral apron, integral flange +- Left-hand drain +- Nominal Size: 60"" x 32"" x 19"" +- Bathing Well: 45 1/8"" x 19 11/16"" x 15"" +- 58 gallon capacity +- Requires ClearFlo drain (K-7272) +- Therapy options available",Biscuit,K-1123-LA-96,"$1,097.80",Granite Group,0.6,https://www.us.kohler.com/us/archer-60-x-32-alcove-bath-with-integral-apron-integral-flange-and-left-hand-drain/productDetail/bathing/425546.htm?skuId=394480,,"=IMAGE(""https://kohler.scene7.com/is/image/PAWEB/Category_Template?$PDPcon$&$gradient_src=PAWEB%2Forganic-gradient&$shadow_src=PAWEB%2FBlank&$Badge1_src=PAWEB%2FBlank&$Badge4_src=PAWEB%2FBlank&$Badge3_src=PAWEB%2FBlank&$Badge2_src=PAWEB%2FBlank&$product_src=is{PAWEB%2Fzac32884_rgb}&fmt"",1)", +Kohler Archer 66x32x19 (Acrylic / Alcove / LH),"Archer acrylic alcove bath with integral apron, integral flange and left-hand drain | Nominal Size: 60"" x 32"" x 19"" | Bathing Well: 45 1/8"" x 19 11/16"" x 15"" (58 gallon capacity); Requires ClearFlo drain (K-7272)",White,K-1948-LA-0,"$1,297.45",Granite Group,0.6,https://www.us.kohler.com/us/archer-66-x-32-three-side-integral-flange-bath-w-left-hand-drain/productDetail/bathing/931171.htm?skuId=931012&brandId=1159437,,"=IMAGE(""https://kohler.scene7.com/is/image/PAWEB/Category_Template?$PDPcon$&$gradient_src=PAWEB%2Forganic-gradient&$shadow_src=PAWEB%2FBlank&$Badge1_src=PAWEB%2FBlank&$Badge4_src=PAWEB%2FBlank&$Badge3_src=PAWEB%2FBlank&$Badge2_src=PAWEB%2FBlank&$product_src=is{PAWEB%2Fzac32876_rgb}"",1)", +Kohler Archer 66x32x19 (Acrylic / Alcove / LH),"Archer acrylic alcove bath with integral flange and left-hand drain | Nominal Size: 66"" x 32"" x 19"" | Bathing Well: 50 3/8"" x 19 5/8"" x 15 5/16"" (66 gallon capacity); Requires ClearFlo drain (K-7272)",White,K-1948-LA-0,"$1,297.45",Granite Group,0.6,https://www.us.kohler.com/us/archer-exocrylic-66-x-32-three-side-integral-flange-bath-with-left-hand-drain/productDetail/bathing/931171.htm,,"=IMAGE(""https://kohler.scene7.com/is/image/PAWEB/Category_Template?$PDPcon$&$gradient_src=PAWEB%2Forganic-gradient&$shadow_src=PAWEB%2FBlank&$Badge1_src=PAWEB%2FBlank&$Badge4_src=PAWEB%2FBlank&$Badge3_src=PAWEB%2FBlank&$Badge2_src=PAWEB%2FBlank&$product_src=is{PAWEB%2Fzac32876_rgb}"",1)", +Kohler Archer 66x32x19 (Acrylic / Alcove / RH),"Archer acrylic alcove bath with integral flange and right-hand drain | Nominal Size: 66"" x 32"" x 19"" | Bathing Well: 50 3/8"" x 19 5/8"" x 15 5/16"" (66 gallon capacity); Requires ClearFlo drain (K-7272)",White,K-1948-RA-0,"$1,297.45",Granite Group,0.6,https://www.us.kohler.com/us/archer-66-x-32-three-side-integral-flange-bath-w-right-hand-drain/productDetail/bathing/931172.htm,,"=IMAGE(""https://kohler.scene7.com/is/image/PAWEB/Category_Template?$PDPcon$&$gradient_src=PAWEB%2Forganic-gradient&$shadow_src=PAWEB%2FBlank&$Badge1_src=PAWEB%2FBlank&$Badge4_src=PAWEB%2FBlank&$Badge3_src=PAWEB%2FBlank&$Badge2_src=PAWEB%2FBlank&$product_src=is{PAWEB%2Fzac32878_rgb}"",1)", +Kohler Archer 72x36x19 (Acrylic / Drop-In / End),"Archer® acrylic drop-in bath +- End drain +- Nominal Size: 72"" x 36"" x 19"" +- Bathing Well: 51 3/8"" x 21"" x 15"" +- 72 gallon capacity +- Requires slotted drain (K-7272) +- Therapy options available",White,K-1125,"$1,266.90",Granite Group,0.6,https://www.us.kohler.com/us/archer-72-x-36-drop-in-bath/productDetail/bathing/425557.htm?skuId=394621&brandId=1159437,,"=IMAGE(""https://kohler.scene7.com/is/image/PAWEB/Category_Template?$PDPcon$&$gradient_src=PAWEB%2Forganic-gradient&$shadow_src=PAWEB%2FBlank&$Badge1_src=PAWEB%2FBlank&$Badge4_src=PAWEB%2FBlank&$Badge3_src=PAWEB%2FBlank&$Badge2_src=PAWEB%2FBlank&$product_src=is{PAWEB%2Fzaa57365_rgb}"",1)", +Kohler Bellwether 60x30x14 (Cast Iron / Alcove / LH),"Bellwether® enameled cast iron alcove bath with integral apron, integral flange +- Left-hand drain +- Nominal Size: 60"" x 30"" x 14"" +- Bathing Well: 48 3/4"" x 19 5/16"" x 9"" +- 32.6 gallon capacity +- Requires cable drain (K-37386-NA)",White,K-837-0,$848.00,Granite Group,0.6,https://www.us.kohler.com/us/bellwether-60-x-30-alcove-bath-with-integral-apron-and-left-hand-drain/productDetail/bathing/429144.htm?skuId=415310&brandId=656446,,"=IMAGE(""https://kohler.scene7.com/is/image/PAWEB/Category_Template?$PDPcon$&$gradient_src=PAWEB%2Forganic-gradient&$shadow_src=PAWEB%2FBlank&$Badge1_src=PAWEB%2FBlank&$Badge4_src=PAWEB%2FBlank&$Badge3_src=PAWEB%2FBlank&$Badge2_src=PAWEB%2FBlank&$product_src=is{PAWEB%2Fzaa98489_rgb}"",1)", +Kohler Bellwether 60x30x14 (Cast Iron / Alcove / RH),"Bellwether® enameled cast iron alcove bath with integral apron, integral flange +- Right-hand drain +- Nominal Size: 60"" x 30"" x 14"" +- Bathing Well: 48 3/4"" x 19 5/16"" x 9"" +- 32.6 gallon capacity +- Requires cable drain (K-37386-NA)",White,K-838-0,$848.00,Granite Group,0.6,https://www.us.kohler.com/us/bellwether-60-x-30-alcove-bath-with-integral-apron-and-right-hand-drain/productDetail/bathing/429145.htm,,"=IMAGE(""https://kohler.scene7.com/is/image/PAWEB/Category_Template?$PDPcon$&$gradient_src=PAWEB%2Forganic-gradient&$shadow_src=PAWEB%2FBlank&$Badge1_src=PAWEB%2FBlank&$Badge4_src=PAWEB%2FBlank&$Badge3_src=PAWEB%2FBlank&$Badge2_src=PAWEB%2FBlank&$product_src=is{PAWEB%2Fzaa98506_rgb}"",1)", +Kohler Bellwether 60x32x15 (Cast Iron / Alcove / LH),"Bellwether® enameled cast iron alcove bath with integral apron, integral flange +- Left-hand drain +- Nominal Size: 60"" x 32"" x 15"" +- Bathing Well: 43 7/8"" x 21"" x 11 5/16"" +- 44 gallon capacity +- Requires cable drain (K-37383-NA)",White,"K-875-0","$1,135.40",Granite Group,0.6,https://www.us.kohler.com/us/bellwether-60-x-32-alcove-bath-with-integral-apron-and-left-hand-drain/productDetail/bathing/427014.htm,,"=IMAGE(""https://kohler.scene7.com/is/image/PAWEB/Category_Template?$PDPcon$&$gradient_src=PAWEB%2Forganic-gradient&$shadow_src=PAWEB%2FBlank&$Badge1_src=PAWEB%2FBlank&$Badge4_src=PAWEB%2FBlank&$Badge3_src=PAWEB%2FBlank&$Badge2_src=PAWEB%2FBlank&$product_src=is{PAWEB%2Fzaa70169_rgb}"",1)", +Kohler Bellwether 60x32x15 (Cast Iron / Alcove / RH),"Bellwether® enameled cast iron alcove bath with integral apron, integral flange +- Right-hand drain +- Nominal Size: 60"" x 32"" x 15"" +- Bathing Well: 43 7/8"" x 21"" x 11 5/16"" +- 44 gallon capacity +- Requires cable drain (K-37383-NA)",White,"K-876-0","$1,135.40",Granite Group,0.6,https://www.us.kohler.com/us/bellwether-60-x-32-alcove-bath-with-integral-apron-and-right-hand-drain/productDetail/bathing/427015.htm,,"=IMAGE(""https://kohler.scene7.com/is/image/PAWEB/Category_Template?$PDPcon$&$gradient_src=PAWEB%2Forganic-gradient&$shadow_src=PAWEB%2FBlank&$Badge1_src=PAWEB%2FBlank&$Badge4_src=PAWEB%2FBlank&$Badge3_src=PAWEB%2FBlank&$Badge2_src=PAWEB%2FBlank&$product_src=is{PAWEB%2Fzaa57530_rgb}"",1)", +Kohler Bellwether 66x32x17 (Cast Iron / Alcove / LH),"Bellwether® enameled cast iron alcove bath with integral apron, integral flange, and left-hand drain | Nominal Size: 66"" x 32"" x 17"" | Bathing Well: 50 3/4"" x 20 5/16"" x 11 11/16"" (49 gallon capacity); Requires PureFlo cable drain (K-37386-NA)",White,K-847-0,"$2,282.00",Granite Group,0.6,https://www.us.kohler.com/us/bellwether-66-x-32-alcove-bath-w-integral-apron-and-left-hand-drain/productDetail/bathing/866299.htm?skuId=866260&brandId=656446,,"=IMAGE(""https://kohler.scene7.com/is/image/PAWEB/Category_Template?$PDPcon$&$gradient_src=PAWEB%2Forganic-gradient&$shadow_src=PAWEB%2FBlank&$Badge1_src=PAWEB%2FBlank&$Badge4_src=PAWEB%2FBlank&$Badge3_src=PAWEB%2FBlank&$Badge2_src=PAWEB%2FBlank&$product_src=is{PAWEB%2Fzab20172_rgb}"",1)", +Kohler Bellwether 66x32x17 (Cast Iron / Alcove / RH),"Bellwether® enameled cast iron alcove bath with integral apron, integral flange, and right-hand drain | Nominal Size: 66"" x 32"" x 17"" | Bathing Well: 50 3/4"" x 20 5/16"" x 11 11/16"" (49 gallon capacity); Requires PureFlo cable drain (K-37386-NA)",White,K-848-0,"$2,282.00",Granite Group,0.6,https://www.us.kohler.com/us/bellwether-66-x-32-alcove-bath-w-integral-apron-and-right-hand-drain/productDetail/bathing/866301.htm,,"=IMAGE(""https://kohler.scene7.com/is/image/PAWEB/Category_Template?$PDPcon$&$gradient_src=PAWEB%2Forganic-gradient&$shadow_src=PAWEB%2FBlank&$Badge1_src=PAWEB%2FBlank&$Badge4_src=PAWEB%2FBlank&$Badge3_src=PAWEB%2FBlank&$Badge2_src=PAWEB%2FBlank&$product_src=is{PAWEB%2Fzab20174_rgb}"",1)", +Kohler Mariposa 60x36x20 (Acrylic / Alcove / LH),"Mariposa® acrylic alcove bath with integral apron, integral flange, and left-hand drain | Nominal Size: 60"" x 36"" x 20"" | Bathing Well: 46"" x 24"" x 14"" (63 gallon capacity); Requirs PureFlo drain (K-37383-NA)",White,K-1242-LA-0,"$1,264.90",Granite Group,0.6,https://www.us.kohler.com/us/mariposa-60-x-36-alcove-bath-with-integral-apron-integral-flange-and-left-hand-drain/productDetail/bathing/416183.htm?skuId=367290&brandId=656470,,"=IMAGE(""https://kohler.scene7.com/is/image/PAWEB/Category_Template?$PDPcon$&$gradient_src=PAWEB%2Forganic-gradient&$shadow_src=PAWEB%2FBlank&$Badge1_src=PAWEB%2FBlank&$Badge4_src=PAWEB%2FBlank&$Badge3_src=PAWEB%2FBlank&$Badge2_src=PAWEB%2FBlank&$product_src=is{PAWEB%2Fzaa71457_rgb}"",1)", +Kohler Mariposa 60x36x20 (Acrylic / Alcove / RH),"Mariposa® acrylic alcove bath with integral apron, integral flange, and right-hand drain | Nominal Size: 60"" x 36"" x 20"" | Bathing Well: 46"" x 24"" x 14"" (63 gallon capacity); Requirs PureFlo drain (K-37383-NA)",White,K-1242-RA-0,"$1,264.90",Granite Group,0.6,https://www.us.kohler.com/us/mariposa-60-x-36-alcove-bath-with-integral-apron-integral-flange-and-right-hand-drain/productDetail/bathing/416185.htm?skuId=367300&brandId=656470,,"=IMAGE(""https://kohler.scene7.com/is/image/PAWEB/Category_Template?$PDPcon$&$gradient_src=PAWEB%2Forganic-gradient&$shadow_src=PAWEB%2FBlank&$Badge1_src=PAWEB%2FBlank&$Badge4_src=PAWEB%2FBlank&$Badge3_src=PAWEB%2FBlank&$Badge2_src=PAWEB%2FBlank&$product_src=is{PAWEB%2Fzaa76226_rgb}"",1)", +Kohler Mariposa 66x36x20 (Acrylic / Alcove / LH),"Mariposa® acrylic alcove bath with integral apron, integral flange, and left-hand drain | Nominal Size: 66"" x 36"" x 20"" | Bathing Well: 48"" x 24"" x 14"" (67 gallon capacity); Requirs PureFlo drain (K-37383-NA)",White,K-1229-LA-0,"$1,304.40",Granite Group,0.6,https://www.us.kohler.com/us/mariposa-66-x-36-alcove-bath-w-integral-apron-and-left-hand-drain/productDetail/bathing/423834.htm?skuId=387495&brandId=656470,,"=IMAGE(""https://kohler.scene7.com/is/image/PAWEB/Category_Template?$PDPcon$&$gradient_src=PAWEB%2Forganic-gradient&$shadow_src=PAWEB%2FBlank&$Badge1_src=PAWEB%2FBlank&$Badge4_src=PAWEB%2FBlank&$Badge3_src=PAWEB%2FBlank&$Badge2_src=PAWEB%2FBlank&$product_src=is{PAWEB%2Fzaa58838_rgb}"",1)", +Kohler Mariposa 66x36x20 (Acrylic / Alcove / RH),"Mariposa® acrylic alcove bath with integral apron, integral flange, and right-hand drain | Nominal Size: 66"" x 36"" x 20"" | Bathing Well: 48"" x 24"" x 14"" (67 gallon capacity); Requirs PureFlo drain (K-37383-NA)",White,K-1229-RA-0,"$1,304.40",Granite Group,0.6,https://www.us.kohler.com/us/mariposa-66-x-36-alcove-bath-w-integral-apron-and-right-hand-drain/productDetail/bathing/423835.htm,,"=IMAGE(""https://kohler.scene7.com/is/image/PAWEB/Category_Template?$PDPcon$&$gradient_src=PAWEB%2Forganic-gradient&$shadow_src=PAWEB%2FBlank&$Badge1_src=PAWEB%2FBlank&$Badge4_src=PAWEB%2FBlank&$Badge3_src=PAWEB%2FBlank&$Badge2_src=PAWEB%2FBlank&$product_src=is{PAWEB%2Fzaa94360_rgb}"",1)", +Kohler Mariposa 72x36x20 (Acrylic / Drop-In / End),"Mariposa® acrylic drop-in bath +- End drain +- Nominal Size: 72"" x 36"" x 20"" +- Bathing Well: 50"" x 24"" x 14"" +- 72 gallon capacity +- Requires ClearFlo bath drain (K-7259) +- Therapy options available",White,K-1259-0,"$1,143.05",Granite Group,0.6,https://www.us.kohler.com/us/mariposa-72-x-36-drop-in-bath-with-reversible-drain/productDetail/bathing/416225.htm?skuId=355670&brandId=656470,,"=IMAGE(""https://kohler.scene7.com/is/image/PAWEB/Category_Template?$PDPcon$&$gradient_src=PAWEB%2Forganic-gradient&$shadow_src=PAWEB%2FBlank&$Badge1_src=PAWEB%2FBlank&$Badge4_src=PAWEB%2FBlank&$Badge3_src=PAWEB%2FBlank&$Badge2_src=PAWEB%2FBlank&$product_src=is{PAWEB%2Fzaa71506_rgb}"",1)", +Kohler Stargaze 72x36x25 (Acrylic / Freestanding / Centered),"Stargaze Acrylic freestanding tub +- Center Drain +- Nominal Size: 72"" x 36"" x 25"" +- Bathing Well: 50"" x 25 1/2"" x 17"" +- 90 Gallon Capacity +- Requires K-7265 Brass Toe Tap Drain or K-37385 Rough in Cable Drain",White,K-6366,"$3,798.00",Granite Group,0.6,https://www.us.kohler.com/us/stargaze-72-x-36-freestanding-bath-with-straight-shroud-and-center-drain/productDetail/bathing/961643.htm?skuId=961623&brandId=964943,,"=IMAGE(""https://kohler.scene7.com/is/image/PAWEB/Category_Template?$PDPcon$&$gradient_src=PAWEB%2Forganic-gradient&$shadow_src=PAWEB%2FBlank&$Badge1_src=PAWEB%2FBlank&$Badge4_src=PAWEB%2FBlank&$Badge3_src=PAWEB%2FBlank&$Badge2_src=PAWEB%2FBlank&$product_src=is{PAWEB%2Fzab30804_rgb}&fmt"",1)", +Kohler Sunstruck 66x36x24 (Acrylic / Freestanding / Center),"Sunstruck® acrylic oval freestanding bath with straight shroud, center drain, and wide ledge for faucet installation | Nominal Size: 66"" x 36"" x 24"" | Bathing Well: 45 3/4"" 24 7/8"" 16"" (72.6 gallon capacity); Therapy systems available; Requires ClearFlo toe-tap drain (K-7265)",White,K-6368-0,"$3,551.40",Granite Group,0.6,https://www.us.kohler.com/us/sunstruck-66-x-36-oval-freestanding-bath-with-straight-shroud-and-center-drain/productDetail/bathing/961645.htm?skuId=961627&brandId=964942,,"=IMAGE(""https://kohler.scene7.com/is/image/PAWEB/Category_Template?$PDPcon$&$gradient_src=PAWEB%2Forganic-gradient&$shadow_src=PAWEB%2FBlank&$Badge1_src=PAWEB%2FBlank&$Badge4_src=PAWEB%2FBlank&$Badge3_src=PAWEB%2FBlank&$Badge2_src=PAWEB%2FBlank&$product_src=is{PAWEB%2Fzab30768_rgb}"",1)", +Kohler Underscore 60x30x20 (Acrylic / Alcove / LH),"Underscore® acrylic alcove bath with integral apron, integral flange, and left-hand drain | Nominal Size: 60"" x 30"" x 20"" | Bathing Well: 46 1/4"" x 22"" x 15 1/2"" (68 gallon capacity); Requires ClearFlo slotted drain (K-7272)",White,K-20201-LA-0,"$1,028.55",Granite Group,0.6,https://www.us.kohler.com/us/underscore-60-x-30-alcove-bath-w-integral-apron-integral-flange-and-left-hand-drain/productDetail/bathing/1302655.htm,,"=IMAGE(""https://kohler.scene7.com/is/image/PAWEB/Category_Template?$PDPcon$&$gradient_src=PAWEB%2Forganic-gradient&$shadow_src=PAWEB%2FBlank&$Badge1_src=PAWEB%2FBlank&$Badge4_src=PAWEB%2FBlank&$Badge3_src=PAWEB%2FBlank&$Badge2_src=PAWEB%2FBlank&$product_src=is{PAWEB%2Fzac10581_rgb}"",1)", +Kohler Underscore 60x30x20 (Acrylic / Alcove / RH),"Underscore® acrylic alcove bath with integral apron, integral flange, and right-hand drain | Nominal Size: 60"" x 30"" x 20"" | Bathing Well: 46 1/4"" x 22"" x 15 1/2"" (68 gallon capacity); Requires ClearFlo slotted drain (K-7272)",White,K-20201-RA-0,"$1,028.55",Granite Group,0.6,https://www.us.kohler.com/us/underscore-60-x-30-alcove-bath-w-integral-apron-integral-flange-and-right-hand-drain/productDetail/bathing/1302656.htm?skuId=1302577&brandId=656492,,"=IMAGE(""https://kohler.scene7.com/is/image/PAWEB/Category_Template?$PDPcon$&$gradient_src=PAWEB%2Forganic-gradient&$shadow_src=PAWEB%2FBlank&$Badge1_src=PAWEB%2FBlank&$Badge4_src=PAWEB%2FBlank&$Badge3_src=PAWEB%2FBlank&$Badge2_src=PAWEB%2FBlank&$product_src=is{PAWEB%2Fzac10585_rgb}"",1)", +Kohler Underscore 60x30x22 (Acrylic / Alcove / LH),"Underscore® acrylic alcove bath with integral apron, integral flange, and left-hand drain | Nominal Size: 60"" x 30"" x 22"" | Bathing Well: 46 1/4"" x 22"" x 15 1/2"" (68 gallon capacity); Requires ClearFlo slotted drain (K-7272)",White,K-1956-LA-0,"$1,028.55",Granite Group,0.6,https://www.us.kohler.com/us/underscore-60-x-30-alcove-bath-w-integral-apron-integral-flange-and-left-hand-drain/productDetail/bathing/1267892.htm?skuId=1267798&brandId=656492,,"=IMAGE(""https://kohler.scene7.com/is/image/PAWEB/Category_Template?$PDPcon$&$gradient_src=PAWEB%2Forganic-gradient&$shadow_src=PAWEB%2FBlank&$Badge1_src=PAWEB%2FBlank&$Badge4_src=PAWEB%2FBlank&$Badge3_src=PAWEB%2FBlank&$Badge2_src=PAWEB%2FBlank&$product_src=is{PAWEB%2Fzac10581_rgb}"",1)", +Kohler Underscore 60x30x22 (Acrylic / Alcove / RH),"Underscore® acrylic alcove bath with integral apron, integral flange, and right-hand drain | Nominal Size: 60"" x 30"" x 22"" | Bathing Well: 46 1/4"" x 22"" x 15 1/2"" (68 gallon capacity); Requires ClearFlo slotted drain (K-7272)",White,K-1956-RA-0,"$1,028.55",Granite Group,0.6,https://www.us.kohler.com/us/underscore-60-x-30-alcove-bath-w-integral-apron-integral-flange-and-right-hand-drain/productDetail/bathing/1267893.htm,,"=IMAGE(""https://kohler.scene7.com/is/image/PAWEB/Category_Template?$PDPcon$&$gradient_src=PAWEB%2Forganic-gradient&$shadow_src=PAWEB%2FBlank&$Badge1_src=PAWEB%2FBlank&$Badge4_src=PAWEB%2FBlank&$Badge3_src=PAWEB%2FBlank&$Badge2_src=PAWEB%2FBlank&$product_src=is{PAWEB%2Fzac10585_rgb}"",1)", +Kohler Underscore 60x32x22 (Acrylic / Alcove / LH),"Underscore® acrylic alcove bath with integral apron, integral flange, and left-hand drain | Nominal Size: 60"" x 32"" x 22"" | Bathing Well: 46 1/4"" x 22"" x 15 1/2"" (68 gallon capacity); Requires ClearFlo slotted drain (K-7272)",White,K-1957-LA-0,"$1,028.55",Granite Group,0.6,https://www.us.kohler.com/us/underscore-60-x-32-alcove-bath-w-integral-apron-integral-flange-and-left-hand-drain/productDetail/bathing/1267894.htm,,"=IMAGE(""https://kohler.scene7.com/is/image/PAWEB/Category_Template?$PDPcon$&$gradient_src=PAWEB%2Forganic-gradient&$shadow_src=PAWEB%2FBlank&$Badge1_src=PAWEB%2FBlank&$Badge4_src=PAWEB%2FBlank&$Badge3_src=PAWEB%2FBlank&$Badge2_src=PAWEB%2FBlank&$product_src=is{PAWEB%2Fzac10589_rgb}"",1)", +Kohler Underscore 60x32x22 (Acrylic / Alcove / RH),"Underscore® acrylic alcove bath with integral apron, integral flange, and right-hand drain | Nominal Size: 60"" x 32"" x 22"" | Bathing Well: 46 1/4"" x 22"" x 15 1/2"" (68 gallon capacity); Requires ClearFlo slotted drain (K-7272)",White,K-1957-RA-0,"$1,028.55",Granite Group,0.6,https://www.us.kohler.com/us/underscore-60-x-32-alcove-bath-with-integral-apron-integral-flange-and-right-hand-drain/productDetail/bathing/1267895.htm,,"=IMAGE(""https://kohler.scene7.com/is/image/PAWEB/Category_Template?$PDPcon$&$gradient_src=PAWEB%2Forganic-gradient&$shadow_src=PAWEB%2FBlank&$Badge1_src=PAWEB%2FBlank&$Badge4_src=PAWEB%2FBlank&$Badge3_src=PAWEB%2FBlank&$Badge2_src=PAWEB%2FBlank&$product_src=is{PAWEB%2Fzac10593_rgb}"",1)", +Kohler Underscore 60x32x21 (Acrylic / Undermount or Drop-In / LH or RH),"Underscore® acrylic undermount or drop-in bath +- Nominal Size: 60"" x 32"" x 21"" +- Bathing Well: 42"" x 23 1/2"" x 17 1/8"" +- 81 gallon capacity +- Requires Clearflo slotted drain (K-7272) +- Requires undermount kit (K-590) +- Therapy systems Available",White,K-1130-0,"$1,528.45",Granite Group,0.6,https://www.us.kohler.com/us/underscore-rectangle-60-x-32-drop-in-bath/productDetail/bathing/426400.htm?skuId=401128&brandId=656492,,"=IMAGE(""https://kohler.scene7.com/is/image/PAWEB/Category_Template?$PDPcon$&$gradient_src=PAWEB%2Forganic-gradient&$shadow_src=PAWEB%2FBlank&$Badge1_src=PAWEB%2FBlank&$Badge4_src=PAWEB%2FBlank&$Badge3_src=PAWEB%2FBlank&$Badge2_src=PAWEB%2FBlank&$product_src=is{PAWEB%2Fzaa61104_rgb}"",1)", +Kohler Underscore 66x32x22 (Acrylic / Undermount or Drop-In / LH or RH),"Underscore® acrylic undermount or drop-in bath | Nominal Size: 66"" x 32"" x 22"" | Bathing Well: 44 1/2"" x 22 5/16"" x 18"" (85.8 gal capacity); Requires Clearflo slotted drain (K-7272) and undermount installation kit (K-550); Therapy systems Available",White,K-1821-0,"$1,529.35",Granite Group,0.6,https://www.us.kohler.com/us/underscore-rectangle-66-x-32-drop-in-bath/productDetail/bathing/428728.htm?skuId=412447&brandId=656492,,"=IMAGE(""https://kohler.scene7.com/is/image/PAWEB/Category_Template?$PDPcon$&$gradient_src=PAWEB%2Forganic-gradient&$shadow_src=PAWEB%2FBlank&$Badge1_src=PAWEB%2FBlank&$Badge4_src=PAWEB%2FBlank&$Badge3_src=PAWEB%2FBlank&$Badge2_src=PAWEB%2FBlank&$product_src=is{PAWEB%2Fzaa61104_rgb}"",1)", +Kohler Underscore 66x36x22 (Acrylic / Undermount or Drop-In / LH or RH),"Underscore® acrylic drop-in bath +- Nominal Size: 66"" x 36"" x 22"" +- Bathing Well: 44 1/2"" x 26 1/2"" x 17 3/4"""" +- 102 gallon capacity +- Requires Clearflo slotted drain (K-7272) +- Undermount installation kit (K-591) +- Therapy systems Available",White,K-1136-0,"$1,558.00",Granite Group,0.6,https://www.us.kohler.com/us/underscore-rectangle-66-x-36-drop-in-bath-with-end-drain/productDetail/bathing/426401.htm?skuId=401135&brandId=656492,,"=IMAGE(""https://kohler.scene7.com/is/image/PAWEB/Category_Template?$PDPcon$&$gradient_src=PAWEB%2Forganic-gradient&$shadow_src=PAWEB%2FBlank&$Badge1_src=PAWEB%2FBlank&$Badge4_src=PAWEB%2FBlank&$Badge3_src=PAWEB%2FBlank&$Badge2_src=PAWEB%2FBlank&$product_src=is{PAWEB%2Fzaa87042_rgb}&fmt"",1)", +Kohler Underscore Oval 60x36x22 (Acrylic / Freestanding Oval / Center),"Underscore® Oval acrylic freestanding bath with deck space for faucet installation | Nominal Size: 60"" x 36"" x 22"" | Bathing Well: 43"" x 25 5/8"" x 16 5/16"" (72 gallon capacity); Requires Clearflo slotted drain (K-7272); Therapy systems available",White,K-5701-0,"$3,690.70",Granite Group,0.6,https://www.us.kohler.com/us/underscore-oval-60-x-36-freestanding-bath/productDetail/bathing/1100347.htm?skuId=1100283&brandId=656492,,"=IMAGE(""https://kohler.scene7.com/is/image/PAWEB/Category_Template?$PDPcon$&$gradient_src=PAWEB%2Forganic-gradient&$shadow_src=PAWEB%2FBlank&$Badge1_src=PAWEB%2FBlank&$Badge4_src=PAWEB%2FBlank&$Badge3_src=PAWEB%2FBlank&$Badge2_src=PAWEB%2FBlank&$product_src=is{PAWEB%2Fzab78293_rgb}"",1)", +Kohler Underscore 72x36x23 (Acrylic / Undermount or Drop-in / Center),"Underscore® acrylic undermount or drop-in bath +- Center drain +- Nominal Size: 72"" x 36"" x 23"" +- Bathing Well: 45 3/4"" x 25 15/16"" x 19 7/16"" +- 112.9 gallon capacity +- Requires ClearFlo bath drain (K-7272) +- Requires undermount kit (K-593) +- Therapy options available",White,K-1834-0,"$1,811.55",Granite Group,0.6,https://www.us.kohler.com/us/underscore-rectangle-72-x-36-drop-in-bath-with-center-drain/productDetail/bathing/429124.htm?skuId=415250&brandId=656492,,"=IMAGE(""https://kohler.scene7.com/is/image/PAWEB/Category_Template?$PDPcon$&$gradient_src=PAWEB%2Forganic-gradient&$shadow_src=PAWEB%2FBlank&$Badge1_src=PAWEB%2FBlank&$Badge4_src=PAWEB%2FBlank&$Badge3_src=PAWEB%2FBlank&$Badge2_src=PAWEB%2FBlank&$product_src=is{PAWEB%2Fzab02156_rgb}"",1)", +Kohler Underscore 72x42x23 (Acrylic / Undermount or Drop-in / Center),"Underscore® acrylic undermount or drop-in bath +- Center drain +- Nominal Size: 72"" x 42"" x 23"" +- Bathing Well: 45 3/4"" x 32"" x 19"" +- 139 gallon capacity +- Requires ClearFlo bath drain (K-7272) +- Requires undermount kit (K-592) +- Therapy options available",White,K-1137-0,"$1,858.00",Granite Group,0.6,https://www.us.kohler.com/us/underscore-rectangle-72-x-42-drop-in-bath-with-center-drain/productDetail/bathing/426402.htm?skuId=401151&brandId=656492,,"=IMAGE(""https://kohler.scene7.com/is/image/PAWEB/Category_Template?$PDPcon$&$gradient_src=PAWEB%2Forganic-gradient&$shadow_src=PAWEB%2FBlank&$Badge1_src=PAWEB%2FBlank&$Badge4_src=PAWEB%2FBlank&$Badge3_src=PAWEB%2FBlank&$Badge2_src=PAWEB%2FBlank&$product_src=is{PAWEB%2Fzaa68539_rgb}"",1)", +Kohler Veil 66x36x25 (Lithocast / Freestanding),"Veil™ Lithocast® Gloss cast resin freestanding bath +- Nominal Size: 66"" x 36"" x 25 1/2"" +- Bathing Well: 46 9/16"" x 22 3/8"" x 15 7/8"" +- 84.29 gal capacity +- Includes center toe-tap drain with matching cover",White,K-8331-0,"$6,198.00",Granite Group,0.6,https://www.us.kohler.com/us/veil-66-x-36-freestanding-bath-with-center-toe-tap-drain/productDetail/bathing/1284613.htm?skuId=1284591&brandId=1348405,,"=IMAGE(""https://kohler.scene7.com/is/image/PAWEB/Category_Template?$PDPcon$&$gradient_src=PAWEB%2Forganic-gradient&$shadow_src=PAWEB%2FBlank&$Badge1_src=PAWEB%2FBlank&$Badge4_src=PAWEB%2FBlank&$Badge3_src=PAWEB%2FBlank&$Badge2_src=PAWEB%2FBlank&$product_src=is{PAWEB%2Fzac28844_rgb}"",1)", +Kohler Villager 60x34x14 (Cast Iron / Alcove / LH),"Villager® enameled cast iron alcove bath with integral apron and left-hand drain | Nominal Size: 60"" x 34 1/4"" x 14"" | Bathing Well: 45"" x 22"" x 8 5/8"" (33 gallon capacity); Requires ClearFlo cable drain (K-7213)",White,K-713-0,$821.20,Granite Group,0.6,https://www.us.kohler.com/us/villager-60-x-34-alcove-bath-with-left-hand-drain/productDetail/bathing/419714.htm,,"=IMAGE(""https://kohler.scene7.com/is/image/PAWEB/Category_Template?$PDPcon$&$gradient_src=PAWEB%2Forganic-gradient&$shadow_src=PAWEB%2FBlank&$Badge1_src=PAWEB%2FBlank&$Badge4_src=PAWEB%2FBlank&$Badge3_src=PAWEB%2FBlank&$Badge2_src=PAWEB%2FBlank&$product_src=is{PAWEB%2Fzaa57457_rgb}"",1)", +Kohler Villager 60x34x14 (Cast Iron / Alcove / RH),"Villager® enameled cast iron alcove bath with integral apron and right-hand drain | Nominal Size: 60"" x 34 1/4"" x 14"" | Bathing Well: 45"" x 22"" x 8 5/8"" (33 gallon capacity); Requires ClearFlo cable drain (K-7213)",White,K-714-0,$821.20,Granite Group,0.6,https://www.us.kohler.com/us/villager-60-x-34-alcove-bath-with-right-hand-drain/productDetail/bathing/419715.htm,,"=IMAGE(""https://kohler.scene7.com/is/image/PAWEB/Category_Template?$PDPcon$&$gradient_src=PAWEB%2Forganic-gradient&$shadow_src=PAWEB%2FBlank&$Badge1_src=PAWEB%2FBlank&$Badge4_src=PAWEB%2FBlank&$Badge3_src=PAWEB%2FBlank&$Badge2_src=PAWEB%2FBlank&$product_src=is{PAWEB%2Fzaa63114_rgb}"",1)", +Maax Ariosa 60x32x28 (Acrylic / Freestanding / End),"Ariosa acrylic asymmetrical freestanding bath with deck space for faucet | Nominal Size: 60″ X 32″ X 28″ | Bathing Well: 37 3/4"" x 18 3/8"" x 14 3/4"" (46 gallon capacity); Requires F2 Drain kit (10034341) and linear drain trim (10037670)",White,106266-000-001,"$2,250.00",Robinson,0.6,https://maax.com/en/product/ariosa-6032_106266,,"=IMAGE(""https://abgprdcdn-maax.azureedge.net/-/media/productassets/maax/106266/images/maax-mx106266-001-2.jpg?h=672&w=800&v=1&d=20171018T234113Z&hash=A71A3436CA6179DF6DB753DF5CCDF0A16FCA649E"",1)", +Maax Ariosa 66x36x28 (Acrylic / Freestanding),"Ariosa acrylic freestanding bath with deck space for faucet | Nominal Size: 66"" x 36"" x 28"" | Bathing Well: 41 1/8"" x 18 3/8"" x 15 3/4"" (56 gal capacity); Requires F2 drain kit (10034341) and positioning template (10034336)",White,106267-000-001,"$2,588.00",Robinson,0.6,https://maax.com/en/product/ariosa-6636_106267,,"=IMAGE(""https://abgprdcdn-maax.azureedge.net/-/media/productassets/maax/106267/images/maax-mx106267-001-2.jpg?h=672&w=800&v=1&d=20171018T234145Z&hash=E1FD6161458EDAC61F5B343A2A04FB168778DCEA"",1)", +Maax Bosca 60x30x18 (Acrylic / Alcove / LH),"Bosca IFS acrylic bath with integral apron, integral flange +- Left-hand drain +- Nominal Size: 60"" x 30"" x 18"" +- Bathing Well: 54 3/4"" x 23 3/8"" x 16 3/4"" +- 54 gallon capacity +- Requires separate drain (10022491)",White,106394-L-000-001,$921.00,Robinson,0.6,https://maax.com/en/product/bosca-6030-ifs_106394,,"=IMAGE(""https://maax.com/-/media/productassets/maax/106394/images/maax-mx106394-001-2.jpg"",1)", +Maax Bosca 60x30x18 (Acrylic / Alcove / RH),"Bosca IFS acrylic bath with integral apron, integral flange +- Right-hand drain +- Nominal Size: 60"" x 30"" x 18"" +- Bathing Well: 54 3/4"" x 23 3/8"" x 16 3/4"" +- 54 gallon capacity +- Requires separate drain (10022491)",White,106394-R-000-001,$921.00,Robinson,0.6,https://maax.com/en/product/bosca-6030-ifs_106394?v=106394-R-000-001,,"=IMAGE(""https://maax.com/-/media/productassets/maax/106394/images/maax-mx106394-001-2.jpg"",1)", +Maax Brioso 66x36x23 (Acrylic / Freestanding),"Brioso acrylic freestanding bath with deck space for faucet +- Nominal Size: 66"" x 36"" x 23"" +- Bathing Well: 62 7/8"" x 28"" +- 68 gal capacity +- Requires F2 drain kit (10034341) +- Requires positioning template (10034338)",White,103903-000-002,"$1,928.00",Robinson,0.6,https://maax.com/en/product/brioso-6636_103903,,"=IMAGE(""https://maax.com/-/media/productassets/maax/103903/images/maax-mx103903-002-2.jpg?h=672&w=800&v=1&d=20180228T192229Z&hash=8E4ADE6E5BFAC43CA367792F1ADA0164DCE11660"",1)", +Maax Brioso 66x36x23 (Acrylic / Freestanding) (GRAY),"Brioso acrylic freestanding bath with deck space for faucet +- Nominal Size: 66"" x 36"" x 23"" +- Bathing Well: 62 7/8"" x 28"" +- 68 gal capacity +- Requires F2 drain kit (10034341) +- Requires positioning template (10034338)",White with gray apron,103903-000-010,"$2,228.00",Robinson,0.6,https://maax.com/en/product/brioso-6636_103903,,"=IMAGE(""https://maax.com/-/media/productassets/maax/103903/images/maax-mx103903-010.jpg?h=672&w=800&v=1&d=20171018T213220Z&hash=ACD1FDE58539898DBCE714FB237FCF6651A962BF"",1)", +Maax Brioso 66x36x23 (Acrylic / Freestanding) (BLACK),"Brioso acrylic freestanding bath with deck space for faucet +- Nominal Size: 66"" x 36"" x 23"" +- Bathing Well: 62 7/8"" x 28"" +- 68 gal capacity +- Requires F2 drain kit (10034341) +- Requires positioning template (10034338)",White with black apron,103903-000-015,"$2,228.00",Robinson,0.6,https://maax.com/en/product/brioso-6636_103903,,"=IMAGE(""https://maax.com/-/media/productassets/maax/103903/images/maax-mx103903-015.jpg?h=672&w=800&v=1&d=20171018T213224Z&hash=7443A5E6CDE1DF8CEA9803CC270D5084516E6048"",1)", +Maax Delsia 66x36x27 (Acrylic / Freestanding),"Delsia acrylic freestanding bath with deck space for faucet +- Nominal Size: 66"" x 36"" x 27"" +- Bathing Well: 42"" x 18 7/8"" x 16 3/4"" +- 59 gal capacity +- Requires F2 drain kit (10034341) +- Requires positioning template (10037416)",White,106193-000-002,"$2,162.00",Robinson,0.6,https://maax.com/en/product/delsia-6636_106193,,"=IMAGE(""https://maax.com/-/media/productassets/maax/106193/images/maax-mx106193-002-2.jpg"",1)", +Maax Exhibit 60x30x18 (Acrylic / Alcove / LH),"Exhibit IFS acrylic bath with integral apron, integral flange, and left-hand drain | Nominal Size: 60"" x 30"" x 18"" | Bathing Well: 46 1/4"" x 19 1/2"" x 14 1/4"" (44 gallon capacity); Requires separate drain (10022491); Therapy Systems Available; ",White,105519-L-000-001,$918.00,Robinson,0.6,https://maax.com/en/product/exhibit-6030-ifs_105519?v=105519-L-000-001,,"=IMAGE(""https://abgprdcdn-maax.azureedge.net/-/media/productassets/maax/105519/images/maax-mx105519-001-2.jpg?h=672&w=800&v=1&d=20171018T221041Z&hash=6BC42B32FBCF4138448AA6835BB8F8B7EF4D1BE6"",1)", +Maax Exhibit 60x30x18 (Acrylic / Alcove / RH),"Exhibit IFS acrylic bath with integral apron, integral flange, and right-hand drain | Nominal Size: 60"" x 30"" x 18"" | Bathing Well: 46 1/4"" x 19 1/2"" x 14 1/4"" (44 gallon capacity); Requires separate drain (10022491); Therapy Systems Available; ",White,105519-R-000-001,$918.00,Robinson,0.6,https://maax.com/en/product/exhibit-6030-ifs_105519?v=105519-R-000-001,,"=IMAGE(""https://abgprdcdn-maax.azureedge.net/-/media/productassets/maax/105519/images/maax-mx105519-001-2.jpg?h=672&w=800&v=1&d=20171018T221041Z&hash=6BC42B32FBCF4138448AA6835BB8F8B7EF4D1BE6"",1)", +"Maax Exhibit 60x32x18, No Apron (Acrylic / Alcove / LH)","Exhibit IF acrylic bath with integral flange, less apron +- Left-hand drain +- Nominal Size: 60"" x 32"" x 18"" +- Bathing Well: 44"" x 19 1/8"" x 15 1/8"" +- 48 gallon capacity +- Requires separate drain (10022491) +- Therapy Systems Available",White,105514-L-000-001,$810.00,Robinson,0.6,https://maax.com/en/product/exhibit-6032-if_105514,,"=IMAGE(""https://maax.com/-/media/productassets/maax/105514/images/maax-mx105514-001-2.jpg?h=168&w=200&v=1&d=20171018T220855Z&hash=0DCF08B0964F6CC885098075219585D8C2CC25CC"",1)", +"Maax Exhibit 60x32x18, No Apron (Acrylic / Alcove / RH)","Exhibit IF acrylic bath with integral flange, less apron +- Right-hand drain +- Nominal Size: 60"" x 32"" x 18"" +- Bathing Well: 44"" x 19 1/8"" x 15 1/8"" +- 48 gallon capacity +- Requires separate drain (10022491) +- Therapy Systems Available",White,05514-R-000-001,$810.00,Robinson,0.6,https://maax.com/en/product/exhibit-6032-if_105514?v=105514-R-000-001,,"=IMAGE(""https://maax.com/-/media/productassets/maax/105514/images/maax-mx105514-001-2.jpg?h=168&w=200&v=1&d=20171018T220855Z&hash=0DCF08B0964F6CC885098075219585D8C2CC25CC"",1)", +Maax Exhibit 66x34x18 (Acrylic / Undermount),"Exhibit acrylic bath, less flange, less apron +- Left-hand drain +- Nominal Size: 66"" x 34"" x 18"" +- Bathing Well: 59 1/8"" x 25 7/8"" x 15 1/2"" +- 55 gallon capacity +- Requires separate drain (10022491) +- Therapy Systems Available",White,106219-000-001,$957.00,Robinson,0.6,https://maax.com/en/product/exhibit-6634_106219?__cf_chl_jschl_tk__=ce24468eb66c2381992cacb0f5cf53ebe67c4a7f-1605722559-0-AU_nu2sS6tc9IWooAunD8d1jua8gI9IsOdYHA4iBlV7dpOn3iKHYqKE2_4yXl5gnztr7OiSZKM_wqnW09_TYHqM3lIHQJIvG-PCVkd_ALGc9MdUtI5tnNRX6MFD4Lw-VTseQ9wnj87F5ZGXrktXHMSZByqhzugOLH4moWnZ5-m2BhQ0hWS9xGwMBnuVDxwdw2odVjUE_DSUrv-2LomU3qBVhqfmW2ZPgijHim_PSpywnVg_E7Yx0SxtILUmNvV2tUlPaBR_E6VllxrBOPSu1jALxHHCACwd3DIeuEm896a1RAsURLq7evyWgx20Te00Dk7w4njA3Cgg6VnbzsBXVvNPrhdP-UqUeizNrHSk4VP_z,,"=IMAGE(""https://maax.com/-/media/productassets/maax/106219/images/maax-mx106219-001-2.jpg?h=672&w=800&v=1&d=20171018T232738Z&hash=F2A3D67D85725E3D0AC9D27A0DBDEF42E21A1AED"",1)", +Maax Exhibit 60x32x18 (Acrylic / Alcove / LH),"Exhibit IFS acrylic bath with integral apron, integral flange +- Left-hand drain +- Nominal Size: 60"" x 32"" x 18"" +- Bathing Well: 44"" x 19 1/8"" x 15 1/8"" +- 44 gallon capacity +- Requires separate drain (10022491) +- Therapy Systems Available",White,105520-L-000-001,$992.00,Robinson,0.6,https://maax.com/en/product/exhibit-6032-ifs_105520,,"=IMAGE(""https://maax.com/-/media/productassets/maax/105520/images/maax-mx105520-001-2.jpg"",1)", +Maax Exhibit 60x32x18 (Acrylic / Alcove / RH),"Exhibit IFS acrylic bath with integral apron, integral flange +- Right-hand drain +- Nominal Size: 60"" x 32"" x 18"" +- Bathing Well: 44"" x 19 1/8"" x 15 1/8"" +- 44 gallon capacity +- Requires separate drain (10022491) +- Therapy Systems Available",White,105520-R-000-001,$992.00,Robinson,0.6,https://maax.com/en/product/exhibit-6032-ifs_105520?v=105520-R-000-001,,"=IMAGE(""https://maax.com/-/media/productassets/maax/105520/images/maax-mx105520-001-2.jpg"",1)", +Maax Exhibit 60x36x18 (Acrylic / Alcove / LH),"Exhibit IFS acrylic bath with integral apron, integral flange, and left-hand drain | Nominal Size: 60"" x 36"" x 18"" | Bathing Well: 46 3/8"" x 21 3/4"" x 15 1/2"" (56 gallon capacity); Requires separate drain (10022491); Therapy Systems Available",White,106172-L-000-001,"$1,108.00",Robinson,0.6,https://maax.com/en/product/exhibit-6036-ifs_106172,,"=IMAGE(""https://abgprdcdn-maax.azureedge.net/-/media/productassets/maax/106172/images/maax-mx106172-001-2.jpg?h=672&w=800&v=1&d=20171018T230750Z&hash=AE0AB866C5B4DE41E491EC370CE1FB8EF4547EA9"",1)", +Maax Exhibit 60x36x18 (Acrylic / Alcove / RH),"Exhibit IFS acrylic bath with integral apron, integral flange, and right-hand drain | Nominal Size: 60"" x 36"" x 18"" | Bathing Well: 46 3/8"" x 21 3/4"" x 15 1/2"" (56 gallon capacity); Requires separate drain (10022491); Therapy Systems Available",White,106172-R-000-001,"$1,108.00",Robinson,0.6,https://maax.com/en/product/exhibit-6036-ifs_106172,,"=IMAGE(""https://maax.com/-/media/productassets/maax/106172/images/maax-mx106172-001-2.jpg"",1)", +Maax Exhibit 60x36x18 (Acrylic / Drop In),"Exhibit acrylic drop-in bath +- Nominal Size: 60"" x 36"" x 18"" +- Bathing Well: 54 5/8"" x 27 7/8 x 15 1/2"" +- 41 gallon capacity +- Requires separate drain (10022491) +- Therapy Systems Available",White,106170-000-001,"$1,095.00",Robinson,0.6,https://maax.com/en/product/exhibit-6036_106170,,"=IMAGE(""https://maax.com/-/media/productassets/maax/106170/images/maax-mx106170-001-1.jpg?h=672&w=800&v=1&d=20171018T230647Z&hash=FC0C4EE4D63E341CEE1268A61010235272D159CA"",1)", +Maax Exhibit 66x32x18 (Acrylic / Alcove / LH),"Exhibit IFS acrylic bath with integral apron, integral flange, and left-hand drain | Nominal Size: 66x32x18 | Bathing Well: 46 5/8"" x 19 7/8"" x 14 1/2"" (55 gallon capacity); Requires separate drain (10022491); Therapy Systems Available",White,106221-L-000-001,"$1,152.00",Robinson,0.6,https://maax.com/en/product/exhibit-6632-ifs_106221,,"=IMAGE(""https://maax.com/-/media/productassets/maax/106221/images/maax-mx106221-001-2.jpg?h=672&w=800&v=1&d=20171018T232843Z&hash=59C9B0DB4FCE4F73FF79B541258BB5007E0E7173"",1)", +Maax Exhibit 66x32x18 (Acrylic / Alcove / RH),"Exhibit IFS acrylic bath with integral apron, integral flange, and right-hand drain | Nominal Size: 66x32x18 | Bathing Well: 46 5/8"" x 19 7/8"" x 14 1/2"" (55 gallon capacity); Requires separate drain (10022491); Therapy Systems Available",White,106221-R-000-001,"$1,152.00",Robinson,0.6,https://maax.com/en/product/exhibit-6632-ifs_106221?v=106221-R-000-001,,"=IMAGE(""https://abgprdcdn-maax.azureedge.net/-/media/productassets/maax/106221/images/maax-mx106221-001-2.jpg?h=672&w=800&v=1&d=20171018T232843Z&hash=59C9B0DB4FCE4F73FF79B541258BB5007E0E7173"",1)", +Maax Exhibit 66x36x18 (Acrylic / Alcove / LH),"Exhibit IFS acrylic bath with integral apron, integral flange, and left-hand drain | Nominal Size: 66x36x18 | Bathing Well: 48 5/8"" x 21 7/8"" x 14 1/2"" (60 gallon capacity); Requires separate drain (10022491); Therapy Systems Available",White,106179-L-000-001,"$1,163.00",Robinson,0.6,https://maax.com/en/product/exhibit-6636-ifs_106179?v=106179-L-000-001,,"=IMAGE("""",1)", +Maax Exhibit 66x36x18 (Acrylic / Alcove / RH),"Exhibit IFS acrylic bath with integral apron, integral flange, and right-hand drain | Nominal Size: 66x36x18 | Bathing Well: 48 5/8"" x 21 7/8"" x 14 1/2"" (60 gallon capacity); Requires separate drain (10022491); Therapy Systems Available",White,106179-R-000-001,"$1,163.00",Robinson,0.6,https://maax.com/en/product/exhibit-6636-ifs_106179?v=106179-R-000-001,,"=IMAGE("""",1)", +"Maax Exhibit 72x36x18, No Apron (Acrylic / Alcove / LH)","Exhibit IF acrylic bath with integral flange, less apron +- Left-hand drain +- Nominal Size: 72"" x 36"" x 18"" +- Bathing Well: 66 1/4"" x 27 7/8"" x 14 1/2"" +- 64 gallon capacity +- Requires separate drain (10022491) +- Therapy Systems Available",White,106182-L-000-001,$951.00,Robinson,0.6,https://maax.com/en/product/exhibit-7236-if_106182,,"=IMAGE(""https://maax.com/-/media/productassets/maax/106182/images/maax-mx106182-001-2.jpg?h=672&w=800&v=1&d=20171018T231252Z&hash=358FAC41CFD656EB6F35145D9B09CA57BCA23BD7"",1)", +"Maax Exhibit 72x36x18, No Apron (Acrylic / Alcove / RH)","Exhibit IF acrylic bath with integral flange, less apron +- Right-hand drain +- Nominal Size: 72"" x 36"" x 18"" +- Bathing Well: 66 1/4"" x 27 7/8"" x 14 1/2"" +- 64 gallon capacity +- Requires separate drain (10022491) +- Therapy Systems Available",White,106182-R-000-001,$951.00,Robinson,0.6,https://maax.com/en/product/exhibit-7236-if_106182,,"=IMAGE(""https://maax.com/-/media/productassets/maax/106182/images/maax-mx106182-001-2.jpg?h=672&w=800&v=1&d=20171018T231252Z&hash=358FAC41CFD656EB6F35145D9B09CA57BCA23BD7"",1)", +Maax Jazz F 66x36x24 (Acrylic / Freestanding),"Jazz F acrylic freestanding oval bath +- Nominal Size: 66"" x 36"" x 24"" +- Bathing Well: 57 5/8"" x 24 1/4"" x 18"" +- 58 gallon capacity +- Requires F2 drain (10034341) +- Requires positioning template (10027224) +- Therapy options available",White,105359-000-001,"$2,434.00",Robinson,0.6,https://maax.com/en/product/jazz-f_105359/,,"=IMAGE(""https://maax.com/-/media/productassets/maax/105359/images/maax-mx105359-001-2.jpg?h=672&w=800&v=1&d=20171018T220008Z&hash=7C6AC8354DA32A7A93E6121377618C824677967C"",1)", +Maax Joan 60x32x31 (Acrylic / Freestanding),"Joan acrylic freestanding oval bath | Nominal Size: 61"" x 32"" x 31"" | Bathing Well: 38"" x 15 1/2"" x 16 3/8"" (73 gallon capacity); Requires F2 drain kit (10034341) and positioning template (10037816)",White,106387-000-001,"$2,611.00",Robinson,0.6,https://maax.com/en/product/joan_106387,,"=IMAGE(""https://abgprdcdn-maax.azureedge.net/-/media/productassets/maax/106387/images/maax-mx106387-001-2.jpg?h=672&w=800&v=1&d=20171122T183236Z&hash=4BE07BE4FF82D46127CB72326BA45081FE1EB999"",1)", +Maax Louie 58x29x23 (Acrylic / Freestanding / Center),"Louie acrylic freestanding oval bath +- Nominal Size: 58"" x 29"" x 23"" +- Bathing Well: 40 5/8"" x 16 1/8"" x 15 1/8"" +- 62 gallon capacity +- Requires F2 Drain kit (10034341) and linear drain trim (10037670)",White,106384-000-001,"$2,228.00",Robinson,0.6,https://maax.com/en/product/louie-5829_106384,,"=IMAGE(""https://maax.com/-/media/productassets/maax/106384/images/maax-mx106384-001-2.jpg?h=672&w=800&v=1&d=20171122T183637Z&hash=2B9808991F0FB78FCCED99476FC49C136F5D6738"",1)", +Maax Lounge 64x34x22 (Acrylic / Freestanding),"Lounge acrylic freestanding rectangular bath +- Nominal Size: 64"" x 34"" x 22 1/4"" +- Bathing Well: 57 3/8"" x 32 1/4"" +- 68 gallon capacity +- Requires F2 Drain kit (10034341) +- Requires positioning template (10032737)",White,105798-000-001,"$2,352.00",Robinson,0.6,https://maax.com/en/product/lounge_105798,,"=IMAGE(""https://maax.com/-/media/productassets/maax/105798/images/maax-mx105798-001-2.jpg?h=672&w=800&v=1&d=20171018T223746Z&hash=1C6331D6CF97C26E60FDF008A07F1945D39C4494"",1)", +Maax ModulR 60x32x23 (Acrylic / Alcove / LH),"ModulR acrylic alcove tub with integrated apron, integrated tile flange, and arm rests +- Left-hand drain +- Nominal size: 60"" x 32"" x 23"" +- Bathing well: 45 3/8"" x 17 7/8"" x 16 1/2"" +- Requires waste and overflow (10032807) +- For use with ModulR shower pan",White,"410019-000-001-001 +(No web page; Model pulled from ModulR Configurator Tool)","$1,671.00",Robinson,0.6,NOT AVAILABLE,,"=IMAGE(""https://maax.com/-/media/productassets/maax/410009/images/maax-mx410009-001-1.jpg?h=672&w=800&v=1&d=20171019T010940Z&hash=0DC297BD56EBD162C943F60F83E5C1847E9C2A3D"",1)", +Maax ModulR 60x32x23 (Acrylic / Alcove / RH),"ModulR acrylic alcove tub with integrated apron, integrated tile flange, and arm rests +- Right-hand drain +- Nominal size: 60"" x 32"" x 23"" +- Bathing well: 45 3/8"" x 17 7/8"" x 16 1/2"" +- Requires waste and overflow (10032807) +- For use with ModulR shower pan",White,"410020-000-001-002 +(No web page; Model pulled from ModulR Configurator Tool)","$1,671.00",Robinson,0.6,NOT AVAILABLE,,"=IMAGE(""https://maax.com/-/media/productassets/maax/410009/images/maax-mx410009-001-1.jpg?h=672&w=800&v=1&d=20171019T010940Z&hash=0DC297BD56EBD162C943F60F83E5C1847E9C2A3D"",1)", +Maax ModulR 60x32x18 (Acrylic / Freestanding Wall-Mount / LH),"ModulR acrylic wall mounted tub with integrated apron, integrated flange, armrests, left-hand drain, and deck space for faucet installation | Nominal Size: 60″ X 32″ X 23″ | Bathing Well: 45 3/8"" x 17 7/8"" x 15 3/4"" (58 gallon capacity); Requires flexible waste and overflow (10032807)",White,410009-L-000-001,"$2,044.00",Robinson,0.6,https://maax.com/en/product/modulr-6032-wall-mounted-%28with-armrests%29_410009,,"=IMAGE(""https://abgprdcdn-maax.azureedge.net/-/media/productassets/maax/410009/images/maax-mx410009-001.jpg?h=672&w=800&v=1&d=20171019T010945Z&hash=BF14A82EE8A9ECBFA677A96615CF9047362B77BE"",1)", +Maax ModulR 60x32x18 (Acrylic / Freestanding Wall-Mount / RH),"ModulR acrylic wall mounted tub with integrated apron, integrated flange, armrests, right-hand drain, and deck space for faucet installation | Nominal Size: 60″ X 32″ X 23″ | Bathing Well: 45 3/8"" x 17 7/8"" x 15 3/4"" (58 gallon capacity); Requires flexible waste and overflow (10032807)",White,410009-R-000-001,"$2,044.00",Robinson,0.6,https://maax.com/en/product/modulr-6032-wall-mounted-%28with-armrests%29_410009?v=410009-R-000-001,,"=IMAGE(""https://abgprdcdn-maax.azureedge.net/-/media/productassets/maax/410009/images/maax-mx410009-001.jpg?h=672&w=800&v=1&d=20171019T010945Z&hash=BF14A82EE8A9ECBFA677A96615CF9047362B77BE"",1)", +Maax Optik 60x32x23 (Acrylic / Freestanding Rectangular / End),"Optik acrylic freestanding tub with deck space for faucet installation | Nominal Size: 60″ X 32″ X 23″ | Bathing Well: 43 1/8"" x 19 7/8"" x 17 1/4"" (55 gallon capacity); Requires F2 Drain kit (10034341) and positioning template (10032739); Air massage system available",White,105571-000-001,"$2,250.00",Robinson,0.6,https://maax.com/en/product/optik-6032-f_105571,,"=IMAGE(""https://abgprdcdn-maax.azureedge.net/-/media/productassets/maax/105571/images/maax-mx105571-001-2.jpg?h=672&w=800&v=1&d=20171018T221648Z&hash=5C4AFF642FE89CC1E476ADF60A2E257AB69F6CA8"",1)", +Maax Optik 66x36x24 (Acrylic / Freestanding Rectangular),"Optik acrylic freestanding rectangular bath +- Nominal Size: 66"" x 36"" x 24"" +- Bathing Well: 45"" x 21 5/8"" x 18 1/2"" +- 47 gallon capacity +- Requires F2 drain (10034341) +- Requires positioning template (10028785) +- Therapy options available",White,105742-000-001,"$2,588.00",Robinson,0.6,https://maax.com/en/product/optik-6636-f_105742/,,"=IMAGE(""https://maax.com/-/media/productassets/maax/105742/images/maax-mx105742-001-2.jpg?h=672&w=800&v=1&d=20171018T223244Z&hash=EF2BC25C85F3B68DEF0E1C3A08490BBA1D74234E"",1)", +Maax Optik 66x36x24 (Acrylic / Freestanding Rectangular) (BLACK),"Optik acrylic freestanding rectangular bath +- Nominal Size: 66"" x 36"" x 24"" +- Bathing Well: 45"" x 21 5/8"" x 18 1/2"" +- 47 gallon capacity +- Requires F2 drain (10034341) +- Requires positioning template (10028785) +- Therapy options available",White with black apron,105742-000-015,"$3,105.00",Robinson,0.6,https://maax.com/en/product/optik-6636-f_105742/,,"=IMAGE(""https://maax.com/-/media/productassets/maax/105742/images/maax-mx105742-015-1.jpg?h=672&w=800&v=1&d=20171018T223259Z&hash=2C517D365DCC9DEBF4AF5F935F76F270FDBA1A51"",1)", +Maax Optik 72x36x23 (Acrylic / Undermount / Drop-In / Center),"Optik acrylic undermount or drop-in bath with center drain and deck space for faucet installation +- Nominal Size: 72"" x 36"" x 23 1/2"" +- Bathing Well: 42 1/2"" x 27"" x 20 3/4"" +- 47 gallon capacity +- Requires waste and overflow (10025081 or 535-PVCHK) +- Therapy options available",White,101270-000-001,"$1,567.00",Robinson,0.6,https://maax.com/en/product/optik-7236_101270,,"=IMAGE(""https://maax.com/-/media/productassets/maax/101270/images/maax-mx101270-001-2.jpg?h=672&w=800&v=1&d=20171018T204345Z&hash=008716DF8A3A9DCA6F0A72EB961520944444BB76"",1)", +Maax Optik 72x42x23 (Acrylic / Undermount / Drop-In / Center),"Optik acrylic undermount or drop-in bath with center drain and deck space for faucet installation +- Nominal Size: 72"" x 42"" x 23 1/2"" +- Bathing Well: 42 5/8"" x 33"" x 20 3/4"" +- 92 gallon capacity +- Requires waste and overflow (10025081 or 535-PVCHK) +- Therapy options available",White,101275-000-001,"$1,766.00",Robinson,0.6,https://maax.com/en/product/optik-7242_101275,,"=IMAGE(""https://maax.com/-/media/productassets/maax/101275/images/maax-mx101275-001-2.jpg?h=672&w=800&v=1&d=20171018T204411Z&hash=B760405EFEB919B064C5818AB384564ADA585AB6"",1)", +Maax Pose 60x32x24 (Acrylic / Drop-In or Undermount),"Pose acrylic undermount or drop-in bath with end drain +- Nominal Size: 60"" x 32"" x 24"" +- Bathing Well: 51 1/8"" x 23 5/8"" x 18"" +- 64 gallon capacity +- Requires waste and overflow (10025081 or 535-PVCHK) +- Therapy options available",White,101457-000-001,"$1,019.00",Robinson,0.6,https://maax.com/en/product/pose-6032_101457,,"=IMAGE(""https://maax.com/-/media/productassets/maax/101457/images/maax-mx101457-001-2.jpg?h=168&w=200&v=1&d=20171018T205127Z&hash=0AB1279FB2B428506688F00CAEC0E5631781D263"",1)", +Maax Pose 66x32x24 (Acrylic / Alcove / LH),"Pose IF acrylic alcove bath with integral flange, deck space for faucet installation, and left-hand drain | Nominal Size: 66"" x 32"" x 24"" | Bathing Well: 50 3/8"" x 21 3/8"" x 18"" (72 gallon capacity); Requires cable-operated waste and overflow (10022491); Separate apron available (101954 or 101957); Therapy Systems Available",White,106204-L-000-001,"$1,124.00",Robinson,0.6,https://maax.com/en/product/pose-6632-if_106204,,"=IMAGE(""https://abgprdcdn-maax.azureedge.net/-/media/productassets/maax/106204/images/maax-mx106204-001-2.jpg?h=672&w=800&v=1&d=20171018T232052Z&hash=336A95179742697BF7C957971F1015B1CED760EC"",1)", +Maax Pose 66x32x24 (Acrylic / Alcove / RH),"Pose IF acrylic alcove bath with integral flange, deck space for faucet installation, and right-hand drain | Nominal Size: 66"" x 32"" x 24"" | Bathing Well: 50 3/8"" x 21 3/8"" x 18"" (72 gallon capacity); Requires cable-operated waste and overflow (10022491); Separate apron available (101954 or 101957); Therapy Systems Available",White,106204-R-000-001,"$1,124.00",Robinson,0.6,https://maax.com/en/product/pose-6632-if_106204?v=106204-R-000-001,,"=IMAGE(""https://abgprdcdn-maax.azureedge.net/-/media/productassets/maax/106204/images/maax-mx106204-001-2.jpg?h=672&w=800&v=1&d=20171018T232052Z&hash=336A95179742697BF7C957971F1015B1CED760EC"",1)", +"Maax Pose 66x32x24 (Acrylic / Drop-in, Undermount / LH or RH)","Pose acrylic drop-in or undermount bath with deck space for faucet installation | Nominal Size: 66"" x 32"" x 24"" | Bathing Well: 50 3/8"" x 21 3/8"" x 18"" (72 gal capacity); Requires cable-operated waste and overflow (10022491); Therapy systems available",White,101458-000-001,"$1,124.00",Robinson,0.6,https://maax.com/en/product/pose-6632_101458,,"=IMAGE(""https://abgprdcdn-maax.azureedge.net/-/media/productassets/maax/101458/images/maax-mx101458-001-2.jpg?h=672&w=800&v=1&d=20171018T205158Z&hash=0D4A617683D8E4F068415D63109AFC2859B473B8"",1)", +Maax Pose 66x36x24 (Acrylic / Alcove / LH),"Pose IF acrylic alcove bath with integral flange, deck space for faucet installation, and left-hand drain; Nominal Size: 66"" x 36"" x 24"" | Bathing Well: 50 3/8"" x 25 3/8"" x 18"" (85 gallon capacity); Requires cable-operated waste and overflow (10022491); Separate apron available (101954 or 101957); Therapy Systems Available",White,106207-L-000-001,"$1,211.00",Robinson,0.6,https://maax.com/en/product/pose-6636-if_106207,,"=IMAGE(""https://abgprdcdn-maax.azureedge.net/-/media/productassets/maax/106207/images/maax-mx106207-001-2.jpg?h=672&w=800&v=1&d=20171018T232235Z&hash=240C3322A2EA93AE22D0C27E8A2D4AFE42D8806F"",1)", +Maax Pose 66x36x24 (Acrylic / Alcove / RH),"Pose IF acrylic alcove bath with integral flange, deck space for faucet installation, and right-hand drain; Nominal Size: 66"" x 36"" x 24"" | Bathing Well: 50 3/8"" x 25 3/8"" x 18"" (85 gallon capacity); Requires cable-operated waste and overflow (10022491); Separate apron available (101954 or 101957); Therapy Systems Available",White,106207-R-000-001,"$1,211.00",Robinson,0.6,https://maax.com/en/product/pose-6636-if_106207?v=106207-R-000-001,,"=IMAGE(""https://abgprdcdn-maax.azureedge.net/-/media/productassets/maax/106207/images/maax-mx106207-001-2.jpg?h=672&w=800&v=1&d=20171018T232235Z&hash=240C3322A2EA93AE22D0C27E8A2D4AFE42D8806F"",1)", +Maax Miles 66x36x23 (Acrylic / Freestanding),"Miles acrylic freestanding rectangular bath +- Nominal Size: 66"" x 36"" x 23 1/4"" +- Bathing Well: 43 7/8"" x 27 5/8"" x 17 1/2 +- 50 gallon capacity +- Requires F2 drain (10034341) +- Requires positioning template (10032738) +- Therapy options available",White,105756-000-001,"$2,588.00",Robinson,0.6,https://maax.com/en/product/miles_105756/,,"=IMAGE(""https://maax.com/-/media/productassets/maax/105756/images/maax-mx105756-001-2.jpg?h=672&w=800&v=1&d=20171018T223517Z&hash=1FB400D7EDF534CA74A25E86CE5134B14CB809FC"",1)", +Maax Rubix 60x30x18 (Acrylic / Alcove / LH),"Rubix acrylic alcove bath with integral apron, integral flange, and left-hand drain | Nominal Size: 60"" x 30"" x 18"" | Bathing Well: 49 3/8"" x 21 3/8"" x 15"" (54 gallon capacity); Requires cable-operated waste and overflow (10022491)",White,105815-L-000-001,$921.00,Robinson,0.6,https://maax.com/en/product/rubix-6030_105815,,"=IMAGE(""https://abgprdcdn-maax.azureedge.net/-/media/productassets/maax/105815/images/maax-mx105815-001-2.jpg?h=672&w=800&v=1&d=20171018T223832Z&hash=895F2C69B90F64455264336D8508F550A5BA1195"",1)", +Maax Rubix 60x30x18 (Acrylic / Alcove / RH),"Rubix acrylic alcove bath with integral apron, integral flange, and right-hand drain | Nominal Size: 60"" x 30"" x 18"" | Bathing Well: 49 3/8"" x 21 3/8"" x 15"" (54 gallon capacity); Requires cable-operated waste and overflow (10022491)",White,105815-R-000-001,$921.00,Robinson,0.6,https://maax.com/en/product/rubix-6030_105815,,"=IMAGE(""https://abgprdcdn-maax.azureedge.net/-/media/productassets/maax/105815/images/maax-mx105815-001-2.jpg?h=672&w=800&v=1&d=20171018T223832Z&hash=895F2C69B90F64455264336D8508F550A5BA1195"",1)", +Maax Rubix 60x32x18 (Acrylic / Alcove / LH),"Rubix acrylic alcove bath with integral apron, integral flange +- Left-hand drain +- Nominal Size: 60"" x 32"" x 18"" +- Bathing Well: 54 3/4” x 25 3/8” x 16"" +- 59 gallon capacity +- Requires cable-operated waste and overflow (10022491)",White,105705-L-000-001,$986.00,Robinson,0.6,https://maax.com/en/product/rubix-6032_105705?v=105705-L-000-001,,"=IMAGE(""https://maax.com/-/media/productassets/maax/105705/images/maax-mx105705-001-2.jpg?h=672&w=800&v=1&d=20171018T222709Z&hash=0723938D960B3F4911D73195D44BEA786F1B7746"",1)", +Maax Rubix 60x32x18 (Acrylic / Alcove / RH),"Rubix acrylic alcove bath with integral apron, integral flange +- Right-hand drain +- Nominal Size: 60"" x 32"" x 18"" +- Bathing Well: 54 3/4” x 25 3/8” x 16"" +- 59 gallon capacity +- Requires cable-operated waste and overflow (10022491)",White,105705-R-000-001,$986.00,Robinson,0.6,https://maax.com/en/product/rubix-6032_105705?v=105705-R-000-001,,"=IMAGE(""https://maax.com/-/media/productassets/maax/105705/images/maax-mx105705-001-2.jpg?h=672&w=800&v=1&d=20171018T222709Z&hash=0723938D960B3F4911D73195D44BEA786F1B7746"",1)", +Maax Rubix 66x32x18 (Acrylic / Alcove / LH),"Rubix acrylic alcove bath with integral apron, integral flange, and left-hand drain | Nominal Size: 66"" x 32"" x 18"" | Bathing Well: 55 3/8"" x 23 3/8"" x 15"" (64 gallon capacity); Requires cable-operated waste and overflow (10022491)",White,105735-L-000-001,"$1,217.00",Robinson,0.6,https://maax.com/en/product/rubix-6632_105735?v=105735-L-000-001,,"=IMAGE(""https://abgprdcdn-maax.azureedge.net/-/media/productassets/maax/105735/images/maax-mx105735-001-2.jpg?h=672&w=800&v=1&d=20171018T223216Z&hash=8E8DD5A73BB11F22A6C233E09E58F56546E46763"",1)", +Maax Rubix 66x32x18 (Acrylic / Alcove / RH),"Rubix acrylic alcove bath with integral apron, integral flange, and right-hand drain | Nominal Size: 66"" x 32"" x 18"" | Bathing Well: 55 3/8"" x 23 3/8"" x 15"" (64 gallon capacity); Requires cable-operated waste and overflow (10022491)",White,105735-R-000-001,"$1,217.00",Robinson,0.6,https://maax.com/en/product/rubix-6632_105735?v=105735-R-000-001,,"=IMAGE(""https://abgprdcdn-maax.azureedge.net/-/media/productassets/maax/105735/images/maax-mx105735-001-2.jpg?h=672&w=800&v=1&d=20171018T223216Z&hash=8E8DD5A73BB11F22A6C233E09E58F56546E46763"",1)", +Maax Sax 60x32x24 (Acrylic / Freestanding),"Sax acrylic freestanding oval bath with deck space for faucet +- Nominal Size: 60"" x 32"" x 24 7/8"" +- Bathing Well: 51 3/8"" x 27 1/4"" x 14 3/4"" +- 44 gallon capacity +- Requires F2 drain (10034341) +- Requires positioning template (10032740)",White,105797-000-002,"$1,607.00",Robinson,0.6,https://maax.com/en/product/sax_105797,,"=IMAGE(""https://maax.com/-/media/productassets/maax/105797/images/maax-mx105797-002-2.jpg?h=672&w=800&v=1&d=20180228T192513Z&hash=C763523B26C37DE409DBA4E69DC8436820EA6ECD"",1)", +Maax Villi 65x32x25 (Acrylic / Freestanding / Center),"Villi acrylic freestanding rectangular bath +- Nominal Size: 65"" x 32"" x 25 3/4"" +- Bathing Well: 62"" x 29"" x 15 3/4"" +- 96 gallon capacity +- Requires F2 drain (10034341) +- Requires positioning template (10037817)",White,106388-000-001,"$2,937.00",Robinson,0.6,https://maax.com/en/product/villi_106388,,"=IMAGE(""https://maax.com/-/media/productassets/maax/106388/images/maax-mx106388-001-2.jpg?h=168&w=200&v=1&d=20171122T183203Z&hash=031254AAB52C3BCF1AE9D0085356082CC51A74A4"",1)", +MTI Basics 66x32x19 (Acrylic / Drop-in or Undermount),"Basics® acrylic undermount or drop-in rectangular bath +- Nominal Size: 66"" x 32 1/4"" x 19 1/2"" +- Bathing Well: 48"" x 19"" x 16"" +- 70 gallon capacity +- Requires lift & trun drain (MBWO) +- Therapy options available",White,MBSCR6632,#N/A,Granite Group,0.625,https://mtibaths.com/products/tubs/basics/MBCR6632/,,"=IMAGE(""https://info.mtibaths.com/product-images/products/MBCR6632_oh_h.jpg"",1)", +MTI New Yorker 72x36x24 (Acrylic / Freestanding),"New Yorker acrylic freestanding oval bath with deck space for faucet installation | Nominal Size: 71 3/4"" x 35 1/2"" x 23 1/4"" | Bathing Well: 48 1/2"" x 22 1/2"" x 16 1/2"" (83 gallon capacity); Requires toe-tap drain (SLOTWH); Therapy options available",White,S225-CWH,"$4,640.00",Granite Group,0.625,https://mtibaths.com/products/tubs/designer/225/,,"=IMAGE(""https://info.mtibaths.com/product-images/products/225_s1_h.jpg"",1)", +MTI Reflections 72x36x22 (Acrylic / Drop-In),"Reflections acrylic drop-in bath with back rest +- Nominal Size: 71 1/4"" x 35 3/4"" x 22 1/4"" +- Bathing Well: 44"" x 21"" x 19 1/2"" +- 80 gallon capacity +- Requires cable drain (WOCB) +- Requires flange kit for alcove (TFK3) +- Therapy options available",White,S45,"$1,955.00",Granite Group,0.625,https://mtibaths.com/products/tubs/designer/45/,,"=IMAGE(""https://info.mtibaths.com/product-images/products/45_oh_h.jpg"",1)", +Maidstone Paloma Clawfoot 60x30x21 (Cast Iron / Freestanding),"Paloma cast iron clawfoot bath with no faucet drillings | Nominal Size: 61"" x 30"" x 22 1/2"" | Bathing Well: 41 1/4"" x 18"" x 15"" (58 gallon capacity); Requires toe tapper drain (122POP2)",White with white feet,1201DE60-0-2,"$2,450.00",Robinson ,0.6,https://maidstonesupply.com/bathroom/bathtubs/paloma-cast-iron-clawfoot-tub-no-faucet-drilling.html,,"=IMAGE(""https://maidstonesupply.com/pub/media/catalog/product/cache/6b8245bae7647528e2edd9cb5e275de1/1/2/1201de60-0-2_front.jpg"",1)", +Maidstone Royal Clawfoot 66x30x25(Acryllic/Freestanding/CHR Feet) ,"Royal Acryllic Freestanding Bath With No Faucet Holes l Nominal Size: 66-3/4""x30""x25"" l Bathing Well: 49 Gal. (**DRAIN**)",White(Chrome Feet),1202DE66-0-1,,Robinson ,0.6,https://maidstonesupply.com/bathroom/bathtubs/acrylic-clawfoot-bathtubs/royal-acrylic-clawfoot-tub-rim-faucet-drillings-mws-royal-royal.html,,"=IMAGE(""https://maidstonesupply.com/pub/media/catalog/product/cache/6b8245bae7647528e2edd9cb5e275de1/1/2/1202de60-7-1_front.jpg"",1)", +Maidstone Royal Clawfoot 66x30x25 with WHITE FEET (Acryllic / Freestanding) ,"Royal acrylic freestanding oval bath with 7"" rim for faucet +- Nominal Size: 66-3/4"" x 30"" x 25"" +- Bathing Well: 40 1/2"" x 17"" x 13"" +- 49 gallon capacity +- Adjustable leg levelers +- Requires toe tapper drain (122POP2)",White with white feet,1202DE66-7-2,"$2,025.00",Robinson,0.6,https://maidstonesupply.com/bathroom/bathtubs/acrylic-clawfoot-bathtubs/royal-acrylic-clawfoot-tub-rim-faucet-drillings-mws-royal-royal.htmll,,"=IMAGE(""https://maidstonesupply.com/pub/media/catalog/product/cache/6b8245bae7647528e2edd9cb5e275de1/1/2/1202de60-7-2_front.jpg"",1)", +Maidstone Royal Clawfoot 66x30x25 with BRASS FEET (Acryllic / Freestanding) ,"Royal acrylic freestanding oval bath with 7"" rim for faucet +- Nominal Size: 66-3/4"" x 30"" x 25"" +- Bathing Well: 40 1/2"" x 17"" x 13"" +- 49 gallon capacity +- Adjustable leg levelers +- Requires toe tapper drain (122POP2)",White with brass feet,1202DE66-7-3,"$2,025.00",Robinson,0.6,https://maidstonesupply.com/bathroom/bathtubs/acrylic-clawfoot-bathtubs/royal-acrylic-clawfoot-tub-rim-faucet-drillings-mws-royal-royal.html,,"=IMAGE(""https://maidstonesupply.com/pub/media/catalog/product/cache/6b8245bae7647528e2edd9cb5e275de1/1/2/1202de60-7-3_front.jpg"",1)", +MAIDSTONE Suffolk 57x28x24 with Gold Feet (Acryllic / Freestanding),"Suffolk acrylic freestanding bath with no faucet holes +- Nominal Size: 57-7/8"" x 28"" x 24-1/2"" +- Bathing Well: 32 1/2"" x 14-1/2"" x 13"" +- 43 gallon capacity +- Adjustable leg levelers +- Requires toe tapper drain (122POP2)",White with Brass Feet,1202SL57-0-3,"$1,975.00",Robinson,0.6,https://maidstonesupply.com/suffolk-acrylic-clawfoot-slipper-tub-suffolk.html,,"=IMAGE(""https://maidstonesupply.com/pub/media/catalog/product/cache/6b8245bae7647528e2edd9cb5e275de1/1/2/1202sl54-0-3_front_2_1.jpg"",1)", +MAIDSTONE Suffolk 57x28x24 with White Feet (Acryllic / Freestanding),"Suffolk acrylic freestanding bath with no faucet holes +- Nominal Size: 57-7/8"" x 28"" x 24-1/2"" +- Bathing Well: 32 1/2"" x 14-1/2"" x 13"" +- 43 gallon capacity +- Adjustable leg levelers +- Requires toe tapper drain (122POP2)",White with White Feet,1202SL57-0-2,"$2,025.00",Robinson,0.6,https://maidstonesupply.com/suffolk-acrylic-clawfoot-slipper-tub-suffolk.html,,"=IMAGE(""https://maidstonesupply.com/pub/media/catalog/product/cache/6b8245bae7647528e2edd9cb5e275de1/1/2/1202sl54-0-2_front_2_1.jpg"",1)", +Maidstone Sutton 54x29x25 Clawfoot with Chrome Feet (Acrylic / Freestanding),"Sutton acrylic freestanding clawfoot bath +- Nominal Size: 53 1/4"" x 29"" x 25"" +- Bathing Well: 30"" x 14 1/2"" x 13"" +- 37 gallon capacity +- Requires toe tapper drain (122POP2)",White with Chrome Feet,1202CL54-0-1,"$1,980.00",Robinson,0.6,https://maidstonesupply.com/bathroom/bathtubs/sutton-acrylic-clawfoot-tub-no-faucet-drilling.html,,"=IMAGE(""https://maidstonesupply.com/pub/media/catalog/product/cache/6b8245bae7647528e2edd9cb5e275de1/1/2/1202cl54-0-1_front.jpg"",1)", +Maidstone Sutton 54x29x25 Clawfoot with Brushed Nickel Feet (Acrylic / Freestanding),"Sutton acrylic freestanding clawfoot bath +- Nominal Size: 53 1/4"" x 29"" x 25"" +- Bathing Well: 30"" x 14 1/2"" x 13"" +- 37 gallon capacity +- Requires toe tapper drain (122POP2)",White with Brushed Nickel Feet,1202CL54-0-5,"$1,980.00",Robinson,0.6,https://maidstonesupply.com/bathroom/bathtubs/sutton-acrylic-clawfoot-tub-no-faucet-drilling.html,,"=IMAGE(""https://maidstonesupply.com/pub/media/catalog/product/cache/6b8245bae7647528e2edd9cb5e275de1/1/2/1202cl54-0-1_front.jpg"",1)", +Oceania Sublime 66x34x21 (Acrylic / Drop-in),"Sublime acrylic drop-in rectangular bath +- Nominal Size: 66"" x 34"" x 21"" +- Bathing Well: 50 3/4"" x 20 1/8"" x 15"" +- 60 gallon capacity +- Requires drain (WOCO14) +- Therapy options available",White,SU6634,"$5,809.00",Robinson,0.65,https://www.oceania-attitude.com/data/baths/deck-mount/sublime/845,,"=IMAGE(""https://www.oceania-attitude.com/products/galleries/display/1d4/c8e/941ab42377fb75fa5ddd54a994/SU_sublime_overview_LR.jpg?size=545x351&mtime=1360092527"",1)", +Sterling Ensemble 60x30x16 (Vikrell / Alcove / LH),"Ensemble™ Vikrell alcove bath with integral apron, integral flange, and left-hand Drain | Nominal Size: 60"" x 30"" x 16"" | Bathing Well: 45"" x 20"" x 11 1/4"" (44 gallon capacity); Drain not included",White,71171110-0,$319.80,Granite Group,0.65,https://www.sterlingplumbing.com/product-detail/71171110?skuid=71171110-0,,"=image(""https://kohler.scene7.com/is/image/Kohler/Category_Template?$PDPcon$&$gradient_src=Kohler%2Forganic-gradient&$shadow_src=Kohler%2FBlank&$Badge1_src=Kohler%2FBlank&$Badge4_src=Kohler%2FBlank&$Badge3_src=Kohler%2FBlank&$Badge2_src=Kohler%2FBlank&$product_src=is{Kohler%2Fzac32952_rgb}"",1)", +Sterling Ensemble 60x30x16 (Vikrell / Alcove / RH),"Ensemble™ Vikrell alcove bath with integral apron, integral flange, and right-hand Drain | Nominal Size: 60"" x 30"" x 16"" | Bathing Well: 45"" x 20"" x 11 1/4"" (44 gallon capacity); Drain not included",White,71171120-0,$319.80,Granite Group,0.65,https://www.sterlingplumbing.com/product-detail/71171120?skuid=71171120-0,,"=image(""https://kohler.scene7.com/is/image/Kohler/Category_Template?$PDPcon$&$gradient_src=Kohler%2Forganic-gradient&$shadow_src=Kohler%2FBlank&$Badge1_src=Kohler%2FBlank&$Badge4_src=Kohler%2FBlank&$Badge3_src=Kohler%2FBlank&$Badge2_src=Kohler%2FBlank&$product_src=is{Kohler%2Fzac32955_rgb}"",1)", +Sterling Ensemble 60x32x18 (Vikrell / Alcove / LH),"Ensemble™ Vikrell alcove bath with integral apron, integral flange +- Left-hand Drain +- Nominal Size: 60"" x 32"" x 18"" +- Bathing Well: 42"" x 20"" x 13 3/4"" +- 55 gallon capacity +- Drain not included",White,71121110-0,$444.90,Granite Group,0.65,https://www.sterlingplumbing.com/product-detail/71121110?skuid=71121110-0,,"=IMAGE(""https://kohler.scene7.com/is/image/Kohler/Category_Template?$PDPcon$&$gradient_src=Kohler%2Forganic-gradient&$shadow_src=Kohler%2FBlank&$Badge1_src=Kohler%2FBlank&$Badge4_src=Kohler%2FBlank&$Badge3_src=Kohler%2FBlank&$Badge2_src=Kohler%2FBlank&$product_src=is{Kohler%2Fzac32944_rgb}"",1)", +Sterling Ensemble 60x32x18 (Vikrell / Alcove / RH),"Ensemble™ Vikrell alcove bath with integral apron, integral flange +- Right-hand Drain +- Nominal Size: 60"" x 32"" x 18"" +- Bathing Well: 42"" x 20"" x 13 3/4"" +- 55 gallon capacity +- Drain not included",White,71121120-0,$444.90,Granite Group,0.65,https://www.sterlingplumbing.com/product-detail/71121120?skuid=71121120-0,,"=image(""https://kohler.scene7.com/is/image/Kohler/Category_Template?$PDPcon$&$gradient_src=Kohler%2Forganic-gradient&$shadow_src=Kohler%2FBlank&$Badge1_src=Kohler%2FBlank&$Badge4_src=Kohler%2FBlank&$Badge3_src=Kohler%2FBlank&$Badge2_src=Kohler%2FBlank&$product_src=is{Kohler%2Fzac32948_rgb}"",1)", +Sterling Ensemble 60x36x16 (Vikrell / Alcove / LH),"Ensemble™ Vikrell alcove bath with integral apron, integral flange, and left-hand Drain | Nominal Size: 60"" x 36"" x 16"" | Bathing Well: 40"" x 21"" x 11"" (40 gallon capacity); Drain not included",White,"71101110-0 +",$629.55,Granite Group,0.65,https://www.sterlingplumbing.com/product-detail/71101110?skuid=71101110-0,,"=IMAGE(""https://kohler.scene7.com/is/image/Kohler/Category_Template?$PDPcon$&$gradient_src=Kohler%2Forganic-gradient&$shadow_src=Kohler%2FBlank&$Badge1_src=Kohler%2FBlank&$Badge4_src=Kohler%2FBlank&$Badge3_src=Kohler%2FBlank&$Badge2_src=Kohler%2FBlank&$product_src=is{Kohler%2Fzac51282_rgb}"",1)", +Sterling Ensemble 60x36x16 (Vikrell / Alcove / RH),"Ensemble™ Vikrell alcove bath with integral apron, integral flange, and right-hand Drain | Nominal Size: 60"" x 36"" x 16"" | Bathing Well: 40"" x 21"" x 11"" (40 gallon capacity); Drain not included",White,"71101120-0 +",$629.55,Granite Group,0.65,https://www.sterlingplumbing.com/product-detail/71101120?skuid=71101120-0,,"=IMAGE(""https://kohler.scene7.com/is/image/Kohler/Category_Template?$PDPcon$&$gradient_src=Kohler%2Forganic-gradient&$shadow_src=Kohler%2FBlank&$Badge1_src=Kohler%2FBlank&$Badge4_src=Kohler%2FBlank&$Badge3_src=Kohler%2FBlank&$Badge2_src=Kohler%2FBlank&$product_src=is{Kohler%2Fzac53168_rgb}"",1)", +Sterling Ensemble 60x32x18 Tub & Wall Set (Vikrell / Alcove / LH),"Ensemble™ Vikrell aclove bath & wall set with left-hand drain +- Nominal Size: 60"" x 32"" x 18"" / 54"" walls +- Bathing Well: 42"" x 20"" x 13 3/4"" +- 55 gallon capacity +- Drain not included",White,"71220110-0",$919.25,Granite Group,0.65,https://www.sterlingplumbing.com/product-detail/71220110?skuid=71220110-0,,"=IMAGE(""https://kohler.scene7.com/is/image/Kohler/Category_Template?$PDPcon$&$gradient_src=Kohler%2Forganic-gradient&$shadow_src=Kohler%2FBlank&$Badge1_src=Kohler%2FBlank&$Badge4_src=Kohler%2FBlank&$Badge3_src=Kohler%2FBlank&$Badge2_src=Kohler%2FBlank&$product_src=is{Kohler%2Fzab53689_rgb}"",1)", +Sterling Ensemble 60x32x18 Tub & Wall Set (Vikrell / Alcove / RH),"Ensemble™ Vikrell aclove bath & wall set with right-hand drain +- Nominal Size: 60"" x 32"" x 18"" / 54"" walls +- Bathing Well: 42"" x 20"" x 13 3/4"" +- 55 gallon capacity +- Drain not included",White,"71220120-0 +",$919.25,Granite Group,0.65,https://www.sterlingplumbing.com/product-detail/71220120?skuid=71220120-0,,"=IMAGE(""https://kohler.scene7.com/is/image/Kohler/Category_Template?$PDPcon$&$gradient_src=Kohler%2Forganic-gradient&$shadow_src=Kohler%2FBlank&$Badge1_src=Kohler%2FBlank&$Badge4_src=Kohler%2FBlank&$Badge3_src=Kohler%2FBlank&$Badge2_src=Kohler%2FBlank&$product_src=is{Kohler%2Fzac39119_rgb}"",1)", +Sterling Ensemble 60x32x18 Tub & Wall Set WITH BACKERS (Vikrell / Alcove / LH),"Ensemble™ Vikrell aclove bath & wall set with age-in-place backerboards and left-hand drain | Nominal Size: 60"" x 32"" x 18"" (54"" wall height) | Bathing Well: 42"" x 20"" x 13 3/4"" (55 gallon capacity); Drain not included",White,71220116-0,$960.00,Granite Group,0.65,https://www.sterlingplumbing.com/product-detail/71220116?skuid=71220116-0,,"=IMAGE(""https://kohler.scene7.com/is/image/Kohler/Category_Template?$PDPcon$&$gradient_src=Kohler%2Forganic-gradient&$shadow_src=Kohler%2FBlank&$Badge1_src=Kohler%2FBlank&$Badge4_src=Kohler%2FBlank&$Badge3_src=Kohler%2FBlank&$Badge2_src=Kohler%2FBlank&$product_src=is{Kohler%2Fzab53689_rgb}"",1)", +Sterling Ensemble 60x32x18 Tub & Wall Set WITH BACKERS (Vikrell / Alcove / RH),"Ensemble™ Vikrell aclove bath & wall set with age-in-place backerboards and right-hand drain | Nominal Size: 60"" x 32"" x 18"" (54"" wall height) | Bathing Well: 42"" x 20"" x 13 3/4"" (55 gallon capacity); Drain not included",White,71220126-0,$960.00,Granite Group,0.65,https://www.sterlingplumbing.com/product-detail/71220126?skuid=71220126-0,,"=IMAGE(""https://kohler.scene7.com/is/image/Kohler/Category_Template?$PDPcon$&$gradient_src=Kohler%2Forganic-gradient&$shadow_src=Kohler%2FBlank&$Badge1_src=Kohler%2FBlank&$Badge4_src=Kohler%2FBlank&$Badge3_src=Kohler%2FBlank&$Badge2_src=Kohler%2FBlank&$product_src=is{Kohler%2Fzac39119_rgb}"",1)", +Sterling Spectacle 60x32x23 (Acrylic / Freestanding),"Spectacle™ acrylic oval freestanding bath with overflow and deck space for faucet installation +- Nominal Size: 60 1/4"" x 32 1/4"" x 23"" +- Bathing Well: 39 3/16"" x 20 1/4"" x 17 1/4"" +- 65 gallon capacity) +- Toe-tap drain included (Chrome only)",White,95333-0,"$1,735.00",Granite Group,0.65,https://www.sterlingplumbing.com/product-detail/95333?skuid=95333-0,,"=IMAGE(""https://s7d4.scene7.com/is/image/Kohler/Category_Template?$GridView$&$gradient_src=Kohler%2Forganic%2Dgradient&$shadow_src=Kohler%2FBlank&$Badge1_src=Kohler%2F2New&$Badge4_src=Kohler%2FBlank&$Badge3_src=Kohler%2FBlank&$Badge2_src=Kohler%2FBlank&$product_src=is{Kohler%2Fzac77632_rgb}"",1)", +Sterling Spectacle 66x34x23 (Acrylic / Freestanding) ,"Spectacle™ acrylic oval freestanding bath with overflow and deck space for faucet installation +- Nominal Size: 65 3/4"" x 34 1/4"" x 23 3/16"""" +- Bathing Well: 45 1/4"" x 22 1/2"" +- 79 gallon capacity) +- Toe-tap drain included (Chrome only)",White,95334-0,"$1,925.00",Granite Group,0.65,https://www.sterlingplumbing.com/product-detail/95334?skuid=95334-0,,"=IMAGE(""https://kohler.scene7.com/is/image/Kohler/Category_Template?$PDPcon$&$gradient_src=Kohler%2Forganic-gradient&$shadow_src=Kohler%2FBlank&$Badge1_src=Kohler%2FBlank&$Badge4_src=Kohler%2FBlank&$Badge3_src=Kohler%2FBlank&$Badge2_src=Kohler%2FBlank&$product_src=is{Kohler%2Fzac77638_rgb}"",1)", +Sterling Unwind 59x30x22 (Acrylic / Freestanding),"Unwind™ acrylic oval freestanding bath with overflow +- Nominal Size: 59 1/16"" x 29 1/2"" x 22 7/8"" +- Bathing Well: 37 1/8"" x 15 5/16"" x 14 11/16"" +- 46.87 gallon capacity +- Toe-tap drain included (Chrome only)",White,96129-0,"$1,909.00",Granite Group,0.65,https://www.sterlingplumbing.com/product-detail/96129?skuid=96129-0,,"=IMAGE(""https://kohler.scene7.com/is/image/Kohler/Category_Template?$PDPcon$&$gradient_src=Kohler%2Forganic-gradient&$shadow_src=Kohler%2FBlank&$Badge1_src=Kohler%2F2New&$Badge4_src=Kohler%2FBlank&$Badge3_src=Kohler%2FBlank&$Badge2_src=Kohler%2FBlank&$product_src=is{Kohler%2Fzac81694_rgb}"",1)", +Victoria Albert Pescadero 67x32x26 (Englishcast / Freestanding / LH),"Pescadero Englishcast™ cast Volcanic Limestone™ freestanding bath with rear overflow| Nominal Size: 66 1/4"" x 31 3/8"" x 25 3/4"" | Bathing Well: 52 3/8"" x 23 1/4"" x 14 1/4"" (66 3/4 gallon capacity); Requires dual option drain (K-50-XX)",White,PES-N-LH-SW-OF,"$5,598.00",Robinson,0.58,https://vandabaths.com/us/americas/product/pescadero-7/,,"=image(SUBSTITUTE(""https://drive.google.com/open?id=1WymcZDqw8NU7Mj6Uh950Ix8Yq3MQjdkz"",""https://drive.google.com/open?id="",""https://docs.google.com/uc?export=download&id=""),1)", +Victoria Albert Pescadero 67x32x26 (Englishcast / Freestanding / RH),"Pescadero Englishcast™ cast Volcanic Limestone™ freestanding bath with rear overflow| Nominal Size: 66 1/4"" x 31 3/8"" x 25 3/4"" | Bathing Well: 52 3/8"" x 23 1/4"" x 14 1/4"" (66 3/4 gallon capacity); Requires dual option drain (K-50-XX)",White,PES-N-RH-SW-OF,"$5,598.00",Robinson,0.58,https://vandabaths.com/us/americas/product/pescadero-7/,,"=image(SUBSTITUTE(""https://drive.google.com/open?id=1PF9sLj3q4YehLWtMyl93NOvx2SY-G3h6"",""https://drive.google.com/open?id="",""https://docs.google.com/uc?export=download&id=""),1)", From 89a6cd8de0de4c64bc11dd5a68eabc3b5fb7a01d Mon Sep 17 00:00:00 2001 From: Jacob Date: Tue, 17 May 2022 13:00:03 -0500 Subject: [PATCH 09/19] docs(csv-parse): list breaking changes in v5.x (#336) --- packages/csv-parse/CHANGELOG.md | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/packages/csv-parse/CHANGELOG.md b/packages/csv-parse/CHANGELOG.md index 497bd9af6..6559deb3f 100644 --- a/packages/csv-parse/CHANGELOG.md +++ b/packages/csv-parse/CHANGELOG.md @@ -57,6 +57,18 @@ See [Conventional Commits](https://conventionalcommits.org) for commit guideline # [5.0.0](https://github.com/adaltas/node-csv/compare/csv-parse@4.16.3...csv-parse@5.0.0) (2021-11-15) +See also [CSV package for Node.js version 6 (11/15/2021)](https://www.adaltas.com/en/2021/11/15/csv-version-6/) + +### ⚠ BREAKING CHANGES + +* esm migration ([b5c0d4b](https://github.com/adaltas/node-csv/commit/b5c0d4b191c8b57397808c0922a3f08248506a9f)) + CommonJS consumers must change `require('csv-parse/lib/sync')` to `require('csv-parse/sync')` +* **csv-parse:** rename group_columns_by_name option [74334cf](https://github.com/adaltas/node-csv/commit/74334cf0e85e005a878c0597b3300f4762116a0d) +* **csv-parse:** rename RECORD_INCONSISTENT_FIELDS_LENGTH [7b55f05](https://github.com/adaltas/node-csv/commit/7b55f050df327939efcb65d4e76d27f98c89d925) +* **csv-parse:** rename RECORD_DONT_MATCH_COLUMNS_LENGTH [fb391c9](https://github.com/adaltas/node-csv/commit/fb391c92fa248bda30b816930cac88a5d9026b04) +* **csv-parse:** rename skip_records_with_error [0376af7](https://github.com/adaltas/node-csv/commit/0376af7984caa6726d12980edecccda1bbbbcacc) +* **csv-parse:** rename skip_records_with_empty_values [aa432c1](https://github.com/adaltas/node-csv/commit/aa432c1251327b579ee7f71bd9fd776021ac1f1e) +* **csv-parse:** rename relax to relax_quotes [9fffd50](https://github.com/adaltas/node-csv/commit/9fffd50762e10b3794883c6b3751ad209510f82e) ### Bug Fixes @@ -75,11 +87,8 @@ See [Conventional Commits](https://conventionalcommits.org) for commit guideline * **csv-parse:** objname index ([015b936](https://github.com/adaltas/node-csv/commit/015b936ea42026efa52263a7687f886463263ed8)) * **csv-parse:** skip_line_with_errors used with raw print current buffer (fix [#292](https://github.com/adaltas/node-csv/issues/292)) ([2741990](https://github.com/adaltas/node-csv/commit/27419908b9ce5319307bb6647335d5c07cd1e3a4)) * **csv-parse:** ts type encoding with BufferEncoding ([39a4388](https://github.com/adaltas/node-csv/commit/39a43886904801d47a92a3cb5722409f36020534)) -* esm migration ([b5c0d4b](https://github.com/adaltas/node-csv/commit/b5c0d4b191c8b57397808c0922a3f08248506a9f)) * export ts types in sync ([890bf8d](https://github.com/adaltas/node-csv/commit/890bf8d950c18a05cab5e35a461d0847d9425156)) * replace ts types with typesVersions ([acb41d5](https://github.com/adaltas/node-csv/commit/acb41d5031669f2d582e40da1c80f5fd4738fee4)) -* refactor: rename skip_lines_with_empty_values to skip_records_with_empty_values [aa432c1](https://github.com/adaltas/node-csv/commit/aa432c1251327b579ee7f71bd9fd776021ac1f1e) -* refactor: rename skip_lines_with_error to skip_records_with_error [aa432c1](https://github.com/adaltas/node-csv/commit/aa432c1251327b579ee7f71bd9fd776021ac1f1e) From 8a5eb7dfd31b22217db4fbbc832d707221850785 Mon Sep 17 00:00:00 2001 From: David Worms Date: Thu, 27 Jan 2022 16:30:15 +0100 Subject: [PATCH 10/19] feat: wg stream api --- demo/eslint/.eslintrc.js | 16 + demo/eslint/.gitignore | 1 + demo/eslint/index.js | 5 + demo/eslint/package.json | 19 + packages/csv-generate/dist/cjs/index.cjs | 211 +- packages/csv-generate/dist/cjs/sync.cjs | 211 +- packages/csv-generate/dist/esm/index.js | 221 +- packages/csv-generate/dist/esm/sync.js | 221 +- packages/csv-generate/dist/iife/index.js | 221 +- packages/csv-generate/dist/iife/sync.js | 221 +- packages/csv-generate/dist/umd/index.js | 221 +- packages/csv-generate/dist/umd/sync.js | 221 +- packages/csv-generate/lib/api/index.js | 6 + packages/csv-generate/lib/api/init_state.js | 12 + .../csv-generate/lib/api/normalize_options.js | 63 + packages/csv-generate/lib/api/random.js | 11 + packages/csv-generate/lib/api/read.js | 81 + packages/csv-generate/lib/api/types.js | 25 + packages/csv-generate/lib/index.js | 169 +- packages/csv-generate/lib/stream.js | 23 + packages/csv-generate/package.json | 4 + .../csv-generate/test/api.web_stream.coffee | 52 + .../csv-generate/test/options.columns.coffee | 87 +- .../csv-generate/test/options.seed.coffee | 13 +- packages/csv-parse/dist/cjs/index.cjs | 2207 +-- packages/csv-parse/dist/cjs/sync.cjs | 2186 ++- packages/csv-parse/dist/esm/index.js | 2221 +-- packages/csv-parse/dist/esm/sync.js | 5197 ++----- packages/csv-parse/dist/iife/index.js | 2221 +-- packages/csv-parse/dist/iife/sync.js | 5197 ++----- packages/csv-parse/dist/umd/index.js | 2221 +-- packages/csv-parse/dist/umd/sync.js | 5197 ++----- packages/csv-parse/lib/api/CsvError.js | 19 + packages/csv-parse/lib/api/index.js | 704 + packages/csv-parse/lib/api/init_state.js | 41 + .../lib/api/normalize_columns_array.js | 33 + .../csv-parse/lib/api/normalize_options.js | 438 + packages/csv-parse/lib/index.js | 1233 +- packages/csv-parse/lib/stream.js | 27 + packages/csv-parse/lib/sync.js | 22 +- .../lib/{ => utils}/ResizeableBuffer.js | 1 - packages/csv-parse/lib/utils/is_object.js | 6 + packages/csv-parse/lib/utils/underscore.js | 8 + .../csv-parse/test/ResizableBuffer.coffee | 2 +- .../csv-parse/test/api.assert_error.coffee | 2 +- packages/csv-parse/test/api.types.ts | 4 +- packages/csv-parse/test/api.web_stream.coffee | 54 + packages/csv-parse/test/option.cast.coffee | 2 +- packages/csv-stringify/dist/cjs/index.cjs | 847 +- packages/csv-stringify/dist/cjs/sync.cjs | 875 +- packages/csv-stringify/dist/esm/index.js | 4805 +++---- packages/csv-stringify/dist/esm/sync.js | 3880 +----- packages/csv-stringify/dist/iife/index.js | 11165 ++++++++-------- packages/csv-stringify/dist/iife/sync.js | 3880 +----- packages/csv-stringify/dist/umd/index.js | 11163 +++++++-------- packages/csv-stringify/dist/umd/sync.js | 3880 +----- packages/csv-stringify/lib/api/CsvError.js | 19 + packages/csv-stringify/lib/api/index.js | 219 + .../lib/api/normalize_columns.js | 43 + .../lib/api/normalize_options.js | 184 + packages/csv-stringify/lib/index.js | 538 +- packages/csv-stringify/lib/sync.js | 30 +- packages/csv-stringify/lib/utils/get.js | 82 + packages/csv-stringify/lib/utils/is_object.js | 7 + .../csv-stringify/lib/utils/underscore.js | 9 + packages/csv-stringify/test/api.types.ts | 2 +- .../csv-stringify/test/api.web_stream.coffee | 18 + packages/csv/dist/cjs/index.cjs | 3287 ++--- packages/csv/dist/cjs/sync.cjs | 3284 ++--- packages/csv/dist/esm/index.js | 3289 ++--- packages/csv/dist/esm/sync.js | 3296 ++--- packages/csv/dist/iife/index.js | 3289 ++--- packages/csv/dist/iife/sync.js | 3296 ++--- packages/csv/dist/umd/index.js | 3289 ++--- packages/csv/dist/umd/sync.js | 3296 ++--- .../test/option.parallel.coffee | 4 +- 76 files changed, 41378 insertions(+), 58176 deletions(-) create mode 100644 demo/eslint/.eslintrc.js create mode 100644 demo/eslint/.gitignore create mode 100644 demo/eslint/index.js create mode 100644 demo/eslint/package.json create mode 100644 packages/csv-generate/lib/api/index.js create mode 100644 packages/csv-generate/lib/api/init_state.js create mode 100644 packages/csv-generate/lib/api/normalize_options.js create mode 100644 packages/csv-generate/lib/api/random.js create mode 100644 packages/csv-generate/lib/api/read.js create mode 100644 packages/csv-generate/lib/api/types.js create mode 100644 packages/csv-generate/lib/stream.js create mode 100644 packages/csv-generate/test/api.web_stream.coffee create mode 100644 packages/csv-parse/lib/api/CsvError.js create mode 100644 packages/csv-parse/lib/api/index.js create mode 100644 packages/csv-parse/lib/api/init_state.js create mode 100644 packages/csv-parse/lib/api/normalize_columns_array.js create mode 100644 packages/csv-parse/lib/api/normalize_options.js create mode 100644 packages/csv-parse/lib/stream.js rename packages/csv-parse/lib/{ => utils}/ResizeableBuffer.js (99%) create mode 100644 packages/csv-parse/lib/utils/is_object.js create mode 100644 packages/csv-parse/lib/utils/underscore.js create mode 100644 packages/csv-parse/test/api.web_stream.coffee create mode 100644 packages/csv-stringify/lib/api/CsvError.js create mode 100644 packages/csv-stringify/lib/api/index.js create mode 100644 packages/csv-stringify/lib/api/normalize_columns.js create mode 100644 packages/csv-stringify/lib/api/normalize_options.js create mode 100644 packages/csv-stringify/lib/utils/get.js create mode 100644 packages/csv-stringify/lib/utils/is_object.js create mode 100644 packages/csv-stringify/lib/utils/underscore.js create mode 100644 packages/csv-stringify/test/api.web_stream.coffee diff --git a/demo/eslint/.eslintrc.js b/demo/eslint/.eslintrc.js new file mode 100644 index 000000000..7ec14b54b --- /dev/null +++ b/demo/eslint/.eslintrc.js @@ -0,0 +1,16 @@ +module.exports = { + env: { + browser: false, + commonjs: true, + node: true, + es2021: true, + }, + // Adding airbnb-base throw an error + extends: ['eslint:recommended', 'plugin:import/recommended', 'airbnb-base'], + parserOptions: { + ecmaVersion: 'latest', + }, + rules: { + // 'import/no-unresolved': [2, { commonjs: false }], + }, +}; diff --git a/demo/eslint/.gitignore b/demo/eslint/.gitignore new file mode 100644 index 000000000..09a8422ef --- /dev/null +++ b/demo/eslint/.gitignore @@ -0,0 +1 @@ +!.eslintrc.js diff --git a/demo/eslint/index.js b/demo/eslint/index.js new file mode 100644 index 000000000..e0e0a18fb --- /dev/null +++ b/demo/eslint/index.js @@ -0,0 +1,5 @@ +const { stringify } = require('csv-stringify/sync'); + +const output = stringify([['a', 'b', 'c']]); + +console.log(output); diff --git a/demo/eslint/package.json b/demo/eslint/package.json new file mode 100644 index 000000000..f273cab22 --- /dev/null +++ b/demo/eslint/package.json @@ -0,0 +1,19 @@ +{ + "name": "csv-demo-eslint", + "version": "0.0.1", + "description": "", + "main": "index.js", + "scripts": { + "lint": "eslint ./", + "test": "echo \"Not ready, check output of npm run lint\"" + }, + "license": "MIT", + "dependencies": { + "csv-stringify": "^6.0.5" + }, + "devDependencies": { + "eslint": "^8.11.0", + "eslint-config-airbnb-base": "^15.0.0", + "eslint-plugin-import": "^2.25.4" + } +} diff --git a/packages/csv-generate/dist/cjs/index.cjs b/packages/csv-generate/dist/cjs/index.cjs index c3090e110..ea7dcb621 100644 --- a/packages/csv-generate/dist/cjs/index.cjs +++ b/packages/csv-generate/dist/cjs/index.cjs @@ -10,20 +10,64 @@ function _interopDefaultLegacy (e) { return e && typeof e === 'object' && 'defau var stream__default = /*#__PURE__*/_interopDefaultLegacy(stream); var util__default = /*#__PURE__*/_interopDefaultLegacy(util); -const Generator = function(options = {}){ +const init_state = (options) => { + // State + return { + start_time: options.duration ? Date.now() : null, + fixed_size_buffer: '', + count_written: 0, + count_created: 0, + }; +}; + +// Generate a random number between 0 and 1 with 2 decimals. The function is idempotent if it detect the "seed" option. +const random = function(options={}){ + if(options.seed){ + return options.seed = options.seed * Math.PI * 100 % 100 / 100; + }else { + return Math.random(); + } +}; + +const types = { + // Generate an ASCII value. + ascii: function({options}){ + const column = []; + const nb_chars = Math.ceil(random(options) * options.maxWordLength); + for(let i=0; i { // Convert Stream Readable options if underscored - if(options.high_water_mark){ - options.highWaterMark = options.high_water_mark; + if(opts.high_water_mark){ + opts.highWaterMark = opts.high_water_mark; } - if(options.object_mode){ - options.objectMode = options.object_mode; + if(opts.object_mode){ + opts.objectMode = opts.object_mode; } - // Call parent constructor - stream__default["default"].Readable.call(this, options); // Clone and camelize options - this.options = {}; - for(const k in options){ - this.options[Generator.camelize(k)] = options[k]; + const options = {}; + for(const k in opts){ + options[camelize(k)] = opts[k]; } // Normalize options const dft = { @@ -41,110 +85,100 @@ const Generator = function(options = {}){ sleep: 0, }; for(const k in dft){ - if(this.options[k] === undefined){ - this.options[k] = dft[k]; + if(options[k] === undefined){ + options[k] = dft[k]; } } // Default values - if(this.options.eof === true){ - this.options.eof = this.options.rowDelimiter; + if(options.eof === true){ + options.eof = options.rowDelimiter; } - // State - this._ = { - start_time: this.options.duration ? Date.now() : null, - fixed_size_buffer: '', - count_written: 0, - count_created: 0, - }; - if(typeof this.options.columns === 'number'){ - this.options.columns = new Array(this.options.columns); + if(typeof options.columns === 'number'){ + options.columns = new Array(options.columns); } - const accepted_header_types = Object.keys(Generator).filter((t) => (!['super_', 'camelize'].includes(t))); - for(let i = 0; i < this.options.columns.length; i++){ - const v = this.options.columns[i] || 'ascii'; + const accepted_header_types = Object.keys(types).filter((t) => (!['super_', 'camelize'].includes(t))); + for(let i = 0; i < options.columns.length; i++){ + const v = options.columns[i] || 'ascii'; if(typeof v === 'string'){ if(!accepted_header_types.includes(v)){ throw Error(`Invalid column type: got "${v}", default values are ${JSON.stringify(accepted_header_types)}`); } - this.options.columns[i] = Generator[v]; + options.columns[i] = types[v]; } } - return this; + return options; }; -util__default["default"].inherits(Generator, stream__default["default"].Readable); -// Generate a random number between 0 and 1 with 2 decimals. The function is idempotent if it detect the "seed" option. -Generator.prototype.random = function(){ - if(this.options.seed){ - return this.options.seed = this.options.seed * Math.PI * 100 % 100 / 100; - }else { - return Math.random(); - } -}; -// Stop the generation. -Generator.prototype.end = function(){ - this.push(null); -}; -// Put new data into the read queue. -Generator.prototype._read = function(size){ +const read = (options, state, size, push, close) => { // Already started const data = []; - let length = this._.fixed_size_buffer.length; + let length = state.fixed_size_buffer.length; if(length !== 0){ - data.push(this._.fixed_size_buffer); + data.push(state.fixed_size_buffer); } // eslint-disable-next-line while(true){ // Time for some rest: flush first and stop later - if((this._.count_created === this.options.length) || (this.options.end && Date.now() > this.options.end) || (this.options.duration && Date.now() > this._.start_time + this.options.duration)){ + if((state.count_created === options.length) || (options.end && Date.now() > options.end) || (options.duration && Date.now() > state.start_time + options.duration)){ // Flush if(data.length){ - if(this.options.objectMode){ + if(options.objectMode){ for(const record of data){ - this.__push(record); + push(record); } }else { - this.__push(data.join('') + (this.options.eof ? this.options.eof : '')); + push(data.join('') + (options.eof ? options.eof : '')); } - this._.end = true; + state.end = true; }else { - this.push(null); + close(); } return; } // Create the record let record = []; let recordLength; - this.options.columns.forEach((fn) => { - record.push(fn(this)); + options.columns.forEach((fn) => { + const result = fn({options: options, state: state}); + const type = typeof result; + if(result !== null && type !== 'string' && type !== 'number'){ + throw Error([ + 'INVALID_VALUE:', + 'values returned by column function must be', + 'a string, a number or null,', + `got ${JSON.stringify(result)}` + ].join(' ')); + } + record.push(result); }); // Obtain record length - if(this.options.objectMode){ + if(options.objectMode){ recordLength = 0; // recordLength is currently equal to the number of columns // This is wrong and shall equal to 1 record only - for(const column of record) + for(const column of record){ recordLength += column.length; + } }else { // Stringify the record - record = (this._.count_created === 0 ? '' : this.options.rowDelimiter)+record.join(this.options.delimiter); + record = (state.count_created === 0 ? '' : options.rowDelimiter)+record.join(options.delimiter); recordLength = record.length; } - this._.count_created++; + state.count_created++; if(length + recordLength > size){ - if(this.options.objectMode){ + if(options.objectMode){ data.push(record); for(const record of data){ - this.__push(record); + push(record); } }else { - if(this.options.fixedSize){ - this._.fixed_size_buffer = record.substr(size - length); + if(options.fixedSize){ + state.fixed_size_buffer = record.substr(size - length); data.push(record.substr(0, size - length)); }else { data.push(record); } - this.__push(data.join('')); + push(data.join('')); } return; } @@ -152,42 +186,41 @@ Generator.prototype._read = function(size){ data.push(record); } }; + +const Generator = function(options = {}){ + this.options = normalize_options(options); + // Call parent constructor + stream__default["default"].Readable.call(this, this.options); + this.state = init_state(this.options); + return this; +}; +util__default["default"].inherits(Generator, stream__default["default"].Readable); + +// Stop the generation. +Generator.prototype.end = function(){ + this.push(null); +}; +// Put new data into the read queue. +Generator.prototype._read = function(size){ + const self = this; + read(this.options, this.state, size, function(chunk) { + self.__push(chunk); + }, function(){ + self.push(null); + }); +}; // Put new data into the read queue. Generator.prototype.__push = function(record){ + // console.log('push', record) const push = () => { - this._.count_written++; + this.state.count_written++; this.push(record); - if(this._.end === true){ + if(this.state.end === true){ return this.push(null); } }; this.options.sleep > 0 ? setTimeout(push, this.options.sleep) : push(); }; -// Generate an ASCII value. -Generator.ascii = function(gen){ - // Column - const column = []; - const nb_chars = Math.ceil(gen.random() * gen.options.maxWordLength); - for(let i=0; i { + // State + return { + start_time: options.duration ? Date.now() : null, + fixed_size_buffer: '', + count_written: 0, + count_created: 0, + }; +}; + +// Generate a random number between 0 and 1 with 2 decimals. The function is idempotent if it detect the "seed" option. +const random = function(options={}){ + if(options.seed){ + return options.seed = options.seed * Math.PI * 100 % 100 / 100; + }else { + return Math.random(); + } +}; + +const types = { + // Generate an ASCII value. + ascii: function({options}){ + const column = []; + const nb_chars = Math.ceil(random(options) * options.maxWordLength); + for(let i=0; i { // Convert Stream Readable options if underscored - if(options.high_water_mark){ - options.highWaterMark = options.high_water_mark; + if(opts.high_water_mark){ + opts.highWaterMark = opts.high_water_mark; } - if(options.object_mode){ - options.objectMode = options.object_mode; + if(opts.object_mode){ + opts.objectMode = opts.object_mode; } - // Call parent constructor - stream__default["default"].Readable.call(this, options); // Clone and camelize options - this.options = {}; - for(const k in options){ - this.options[Generator.camelize(k)] = options[k]; + const options = {}; + for(const k in opts){ + options[camelize(k)] = opts[k]; } // Normalize options const dft = { @@ -41,110 +85,100 @@ const Generator = function(options = {}){ sleep: 0, }; for(const k in dft){ - if(this.options[k] === undefined){ - this.options[k] = dft[k]; + if(options[k] === undefined){ + options[k] = dft[k]; } } // Default values - if(this.options.eof === true){ - this.options.eof = this.options.rowDelimiter; + if(options.eof === true){ + options.eof = options.rowDelimiter; } - // State - this._ = { - start_time: this.options.duration ? Date.now() : null, - fixed_size_buffer: '', - count_written: 0, - count_created: 0, - }; - if(typeof this.options.columns === 'number'){ - this.options.columns = new Array(this.options.columns); + if(typeof options.columns === 'number'){ + options.columns = new Array(options.columns); } - const accepted_header_types = Object.keys(Generator).filter((t) => (!['super_', 'camelize'].includes(t))); - for(let i = 0; i < this.options.columns.length; i++){ - const v = this.options.columns[i] || 'ascii'; + const accepted_header_types = Object.keys(types).filter((t) => (!['super_', 'camelize'].includes(t))); + for(let i = 0; i < options.columns.length; i++){ + const v = options.columns[i] || 'ascii'; if(typeof v === 'string'){ if(!accepted_header_types.includes(v)){ throw Error(`Invalid column type: got "${v}", default values are ${JSON.stringify(accepted_header_types)}`); } - this.options.columns[i] = Generator[v]; + options.columns[i] = types[v]; } } - return this; + return options; }; -util__default["default"].inherits(Generator, stream__default["default"].Readable); -// Generate a random number between 0 and 1 with 2 decimals. The function is idempotent if it detect the "seed" option. -Generator.prototype.random = function(){ - if(this.options.seed){ - return this.options.seed = this.options.seed * Math.PI * 100 % 100 / 100; - }else { - return Math.random(); - } -}; -// Stop the generation. -Generator.prototype.end = function(){ - this.push(null); -}; -// Put new data into the read queue. -Generator.prototype._read = function(size){ +const read = (options, state, size, push, close) => { // Already started const data = []; - let length = this._.fixed_size_buffer.length; + let length = state.fixed_size_buffer.length; if(length !== 0){ - data.push(this._.fixed_size_buffer); + data.push(state.fixed_size_buffer); } // eslint-disable-next-line while(true){ // Time for some rest: flush first and stop later - if((this._.count_created === this.options.length) || (this.options.end && Date.now() > this.options.end) || (this.options.duration && Date.now() > this._.start_time + this.options.duration)){ + if((state.count_created === options.length) || (options.end && Date.now() > options.end) || (options.duration && Date.now() > state.start_time + options.duration)){ // Flush if(data.length){ - if(this.options.objectMode){ + if(options.objectMode){ for(const record of data){ - this.__push(record); + push(record); } }else { - this.__push(data.join('') + (this.options.eof ? this.options.eof : '')); + push(data.join('') + (options.eof ? options.eof : '')); } - this._.end = true; + state.end = true; }else { - this.push(null); + close(); } return; } // Create the record let record = []; let recordLength; - this.options.columns.forEach((fn) => { - record.push(fn(this)); + options.columns.forEach((fn) => { + const result = fn({options: options, state: state}); + const type = typeof result; + if(result !== null && type !== 'string' && type !== 'number'){ + throw Error([ + 'INVALID_VALUE:', + 'values returned by column function must be', + 'a string, a number or null,', + `got ${JSON.stringify(result)}` + ].join(' ')); + } + record.push(result); }); // Obtain record length - if(this.options.objectMode){ + if(options.objectMode){ recordLength = 0; // recordLength is currently equal to the number of columns // This is wrong and shall equal to 1 record only - for(const column of record) + for(const column of record){ recordLength += column.length; + } }else { // Stringify the record - record = (this._.count_created === 0 ? '' : this.options.rowDelimiter)+record.join(this.options.delimiter); + record = (state.count_created === 0 ? '' : options.rowDelimiter)+record.join(options.delimiter); recordLength = record.length; } - this._.count_created++; + state.count_created++; if(length + recordLength > size){ - if(this.options.objectMode){ + if(options.objectMode){ data.push(record); for(const record of data){ - this.__push(record); + push(record); } }else { - if(this.options.fixedSize){ - this._.fixed_size_buffer = record.substr(size - length); + if(options.fixedSize){ + state.fixed_size_buffer = record.substr(size - length); data.push(record.substr(0, size - length)); }else { data.push(record); } - this.__push(data.join('')); + push(data.join('')); } return; } @@ -152,42 +186,41 @@ Generator.prototype._read = function(size){ data.push(record); } }; + +const Generator = function(options = {}){ + this.options = normalize_options(options); + // Call parent constructor + stream__default["default"].Readable.call(this, this.options); + this.state = init_state(this.options); + return this; +}; +util__default["default"].inherits(Generator, stream__default["default"].Readable); + +// Stop the generation. +Generator.prototype.end = function(){ + this.push(null); +}; +// Put new data into the read queue. +Generator.prototype._read = function(size){ + const self = this; + read(this.options, this.state, size, function(chunk) { + self.__push(chunk); + }, function(){ + self.push(null); + }); +}; // Put new data into the read queue. Generator.prototype.__push = function(record){ + // console.log('push', record) const push = () => { - this._.count_written++; + this.state.count_written++; this.push(record); - if(this._.end === true){ + if(this.state.end === true){ return this.push(null); } }; this.options.sleep > 0 ? setTimeout(push, this.options.sleep) : push(); }; -// Generate an ASCII value. -Generator.ascii = function(gen){ - // Column - const column = []; - const nb_chars = Math.ceil(gen.random() * gen.options.maxWordLength); - for(let i=0; i { + // State + return { + start_time: options.duration ? Date.now() : null, + fixed_size_buffer: '', + count_written: 0, + count_created: 0, + }; +}; + +// Generate a random number between 0 and 1 with 2 decimals. The function is idempotent if it detect the "seed" option. +const random = function(options={}){ + if(options.seed){ + return options.seed = options.seed * Math.PI * 100 % 100 / 100; + }else { + return Math.random(); + } +}; + +const types = { + // Generate an ASCII value. + ascii: function({options}){ + const column = []; + const nb_chars = Math.ceil(random(options) * options.maxWordLength); + for(let i=0; i { // Convert Stream Readable options if underscored - if(options.high_water_mark){ - options.highWaterMark = options.high_water_mark; + if(opts.high_water_mark){ + opts.highWaterMark = opts.high_water_mark; } - if(options.object_mode){ - options.objectMode = options.object_mode; + if(opts.object_mode){ + opts.objectMode = opts.object_mode; } - // Call parent constructor - Stream.Readable.call(this, options); // Clone and camelize options - this.options = {}; - for(const k in options){ - this.options[Generator.camelize(k)] = options[k]; + const options = {}; + for(const k in opts){ + options[camelize(k)] = opts[k]; } // Normalize options const dft = { @@ -5067,110 +5111,100 @@ const Generator = function(options = {}){ sleep: 0, }; for(const k in dft){ - if(this.options[k] === undefined){ - this.options[k] = dft[k]; + if(options[k] === undefined){ + options[k] = dft[k]; } } // Default values - if(this.options.eof === true){ - this.options.eof = this.options.rowDelimiter; + if(options.eof === true){ + options.eof = options.rowDelimiter; } - // State - this._ = { - start_time: this.options.duration ? Date.now() : null, - fixed_size_buffer: '', - count_written: 0, - count_created: 0, - }; - if(typeof this.options.columns === 'number'){ - this.options.columns = new Array(this.options.columns); + if(typeof options.columns === 'number'){ + options.columns = new Array(options.columns); } - const accepted_header_types = Object.keys(Generator).filter((t) => (!['super_', 'camelize'].includes(t))); - for(let i = 0; i < this.options.columns.length; i++){ - const v = this.options.columns[i] || 'ascii'; + const accepted_header_types = Object.keys(types).filter((t) => (!['super_', 'camelize'].includes(t))); + for(let i = 0; i < options.columns.length; i++){ + const v = options.columns[i] || 'ascii'; if(typeof v === 'string'){ if(!accepted_header_types.includes(v)){ throw Error(`Invalid column type: got "${v}", default values are ${JSON.stringify(accepted_header_types)}`); } - this.options.columns[i] = Generator[v]; + options.columns[i] = types[v]; } } - return this; + return options; }; -util.inherits(Generator, Stream.Readable); -// Generate a random number between 0 and 1 with 2 decimals. The function is idempotent if it detect the "seed" option. -Generator.prototype.random = function(){ - if(this.options.seed){ - return this.options.seed = this.options.seed * Math.PI * 100 % 100 / 100; - }else { - return Math.random(); - } -}; -// Stop the generation. -Generator.prototype.end = function(){ - this.push(null); -}; -// Put new data into the read queue. -Generator.prototype._read = function(size){ +const read = (options, state, size, push, close) => { // Already started const data = []; - let length = this._.fixed_size_buffer.length; + let length = state.fixed_size_buffer.length; if(length !== 0){ - data.push(this._.fixed_size_buffer); + data.push(state.fixed_size_buffer); } // eslint-disable-next-line while(true){ // Time for some rest: flush first and stop later - if((this._.count_created === this.options.length) || (this.options.end && Date.now() > this.options.end) || (this.options.duration && Date.now() > this._.start_time + this.options.duration)){ + if((state.count_created === options.length) || (options.end && Date.now() > options.end) || (options.duration && Date.now() > state.start_time + options.duration)){ // Flush if(data.length){ - if(this.options.objectMode){ + if(options.objectMode){ for(const record of data){ - this.__push(record); + push(record); } }else { - this.__push(data.join('') + (this.options.eof ? this.options.eof : '')); + push(data.join('') + (options.eof ? options.eof : '')); } - this._.end = true; + state.end = true; }else { - this.push(null); + close(); } return; } // Create the record let record = []; let recordLength; - this.options.columns.forEach((fn) => { - record.push(fn(this)); + options.columns.forEach((fn) => { + const result = fn({options: options, state: state}); + const type = typeof result; + if(result !== null && type !== 'string' && type !== 'number'){ + throw Error([ + 'INVALID_VALUE:', + 'values returned by column function must be', + 'a string, a number or null,', + `got ${JSON.stringify(result)}` + ].join(' ')); + } + record.push(result); }); // Obtain record length - if(this.options.objectMode){ + if(options.objectMode){ recordLength = 0; // recordLength is currently equal to the number of columns // This is wrong and shall equal to 1 record only - for(const column of record) + for(const column of record){ recordLength += column.length; + } }else { // Stringify the record - record = (this._.count_created === 0 ? '' : this.options.rowDelimiter)+record.join(this.options.delimiter); + record = (state.count_created === 0 ? '' : options.rowDelimiter)+record.join(options.delimiter); recordLength = record.length; } - this._.count_created++; + state.count_created++; if(length + recordLength > size){ - if(this.options.objectMode){ + if(options.objectMode){ data.push(record); for(const record of data){ - this.__push(record); + push(record); } }else { - if(this.options.fixedSize){ - this._.fixed_size_buffer = record.substr(size - length); + if(options.fixedSize){ + state.fixed_size_buffer = record.substr(size - length); data.push(record.substr(0, size - length)); }else { data.push(record); } - this.__push(data.join('')); + push(data.join('')); } return; } @@ -5178,42 +5212,41 @@ Generator.prototype._read = function(size){ data.push(record); } }; + +const Generator = function(options = {}){ + this.options = normalize_options(options); + // Call parent constructor + Stream.Readable.call(this, this.options); + this.state = init_state(this.options); + return this; +}; +util.inherits(Generator, Stream.Readable); + +// Stop the generation. +Generator.prototype.end = function(){ + this.push(null); +}; +// Put new data into the read queue. +Generator.prototype._read = function(size){ + const self = this; + read(this.options, this.state, size, function(chunk) { + self.__push(chunk); + }, function(){ + self.push(null); + }); +}; // Put new data into the read queue. Generator.prototype.__push = function(record){ + // console.log('push', record) const push = () => { - this._.count_written++; + this.state.count_written++; this.push(record); - if(this._.end === true){ + if(this.state.end === true){ return this.push(null); } }; this.options.sleep > 0 ? setTimeout(push, this.options.sleep) : push(); }; -// Generate an ASCII value. -Generator.ascii = function(gen){ - // Column - const column = []; - const nb_chars = Math.ceil(gen.random() * gen.options.maxWordLength); - for(let i=0; i { + // State + return { + start_time: options.duration ? Date.now() : null, + fixed_size_buffer: '', + count_written: 0, + count_created: 0, + }; +}; + +// Generate a random number between 0 and 1 with 2 decimals. The function is idempotent if it detect the "seed" option. +const random = function(options={}){ + if(options.seed){ + return options.seed = options.seed * Math.PI * 100 % 100 / 100; + }else { + return Math.random(); + } +}; + +const types = { + // Generate an ASCII value. + ascii: function({options}){ + const column = []; + const nb_chars = Math.ceil(random(options) * options.maxWordLength); + for(let i=0; i { // Convert Stream Readable options if underscored - if(options.high_water_mark){ - options.highWaterMark = options.high_water_mark; + if(opts.high_water_mark){ + opts.highWaterMark = opts.high_water_mark; } - if(options.object_mode){ - options.objectMode = options.object_mode; + if(opts.object_mode){ + opts.objectMode = opts.object_mode; } - // Call parent constructor - Stream.Readable.call(this, options); // Clone and camelize options - this.options = {}; - for(const k in options){ - this.options[Generator.camelize(k)] = options[k]; + const options = {}; + for(const k in opts){ + options[camelize(k)] = opts[k]; } // Normalize options const dft = { @@ -5067,110 +5111,100 @@ const Generator = function(options = {}){ sleep: 0, }; for(const k in dft){ - if(this.options[k] === undefined){ - this.options[k] = dft[k]; + if(options[k] === undefined){ + options[k] = dft[k]; } } // Default values - if(this.options.eof === true){ - this.options.eof = this.options.rowDelimiter; + if(options.eof === true){ + options.eof = options.rowDelimiter; } - // State - this._ = { - start_time: this.options.duration ? Date.now() : null, - fixed_size_buffer: '', - count_written: 0, - count_created: 0, - }; - if(typeof this.options.columns === 'number'){ - this.options.columns = new Array(this.options.columns); + if(typeof options.columns === 'number'){ + options.columns = new Array(options.columns); } - const accepted_header_types = Object.keys(Generator).filter((t) => (!['super_', 'camelize'].includes(t))); - for(let i = 0; i < this.options.columns.length; i++){ - const v = this.options.columns[i] || 'ascii'; + const accepted_header_types = Object.keys(types).filter((t) => (!['super_', 'camelize'].includes(t))); + for(let i = 0; i < options.columns.length; i++){ + const v = options.columns[i] || 'ascii'; if(typeof v === 'string'){ if(!accepted_header_types.includes(v)){ throw Error(`Invalid column type: got "${v}", default values are ${JSON.stringify(accepted_header_types)}`); } - this.options.columns[i] = Generator[v]; + options.columns[i] = types[v]; } } - return this; + return options; }; -util.inherits(Generator, Stream.Readable); -// Generate a random number between 0 and 1 with 2 decimals. The function is idempotent if it detect the "seed" option. -Generator.prototype.random = function(){ - if(this.options.seed){ - return this.options.seed = this.options.seed * Math.PI * 100 % 100 / 100; - }else { - return Math.random(); - } -}; -// Stop the generation. -Generator.prototype.end = function(){ - this.push(null); -}; -// Put new data into the read queue. -Generator.prototype._read = function(size){ +const read = (options, state, size, push, close) => { // Already started const data = []; - let length = this._.fixed_size_buffer.length; + let length = state.fixed_size_buffer.length; if(length !== 0){ - data.push(this._.fixed_size_buffer); + data.push(state.fixed_size_buffer); } // eslint-disable-next-line while(true){ // Time for some rest: flush first and stop later - if((this._.count_created === this.options.length) || (this.options.end && Date.now() > this.options.end) || (this.options.duration && Date.now() > this._.start_time + this.options.duration)){ + if((state.count_created === options.length) || (options.end && Date.now() > options.end) || (options.duration && Date.now() > state.start_time + options.duration)){ // Flush if(data.length){ - if(this.options.objectMode){ + if(options.objectMode){ for(const record of data){ - this.__push(record); + push(record); } }else { - this.__push(data.join('') + (this.options.eof ? this.options.eof : '')); + push(data.join('') + (options.eof ? options.eof : '')); } - this._.end = true; + state.end = true; }else { - this.push(null); + close(); } return; } // Create the record let record = []; let recordLength; - this.options.columns.forEach((fn) => { - record.push(fn(this)); + options.columns.forEach((fn) => { + const result = fn({options: options, state: state}); + const type = typeof result; + if(result !== null && type !== 'string' && type !== 'number'){ + throw Error([ + 'INVALID_VALUE:', + 'values returned by column function must be', + 'a string, a number or null,', + `got ${JSON.stringify(result)}` + ].join(' ')); + } + record.push(result); }); // Obtain record length - if(this.options.objectMode){ + if(options.objectMode){ recordLength = 0; // recordLength is currently equal to the number of columns // This is wrong and shall equal to 1 record only - for(const column of record) + for(const column of record){ recordLength += column.length; + } }else { // Stringify the record - record = (this._.count_created === 0 ? '' : this.options.rowDelimiter)+record.join(this.options.delimiter); + record = (state.count_created === 0 ? '' : options.rowDelimiter)+record.join(options.delimiter); recordLength = record.length; } - this._.count_created++; + state.count_created++; if(length + recordLength > size){ - if(this.options.objectMode){ + if(options.objectMode){ data.push(record); for(const record of data){ - this.__push(record); + push(record); } }else { - if(this.options.fixedSize){ - this._.fixed_size_buffer = record.substr(size - length); + if(options.fixedSize){ + state.fixed_size_buffer = record.substr(size - length); data.push(record.substr(0, size - length)); }else { data.push(record); } - this.__push(data.join('')); + push(data.join('')); } return; } @@ -5178,42 +5212,41 @@ Generator.prototype._read = function(size){ data.push(record); } }; + +const Generator = function(options = {}){ + this.options = normalize_options(options); + // Call parent constructor + Stream.Readable.call(this, this.options); + this.state = init_state(this.options); + return this; +}; +util.inherits(Generator, Stream.Readable); + +// Stop the generation. +Generator.prototype.end = function(){ + this.push(null); +}; +// Put new data into the read queue. +Generator.prototype._read = function(size){ + const self = this; + read(this.options, this.state, size, function(chunk) { + self.__push(chunk); + }, function(){ + self.push(null); + }); +}; // Put new data into the read queue. Generator.prototype.__push = function(record){ + // console.log('push', record) const push = () => { - this._.count_written++; + this.state.count_written++; this.push(record); - if(this._.end === true){ + if(this.state.end === true){ return this.push(null); } }; this.options.sleep > 0 ? setTimeout(push, this.options.sleep) : push(); }; -// Generate an ASCII value. -Generator.ascii = function(gen){ - // Column - const column = []; - const nb_chars = Math.ceil(gen.random() * gen.options.maxWordLength); - for(let i=0; i { + // State + return { + start_time: options.duration ? Date.now() : null, + fixed_size_buffer: '', + count_written: 0, + count_created: 0, + }; + }; + + // Generate a random number between 0 and 1 with 2 decimals. The function is idempotent if it detect the "seed" option. + const random = function(options={}){ + if(options.seed){ + return options.seed = options.seed * Math.PI * 100 % 100 / 100; + }else { + return Math.random(); + } + }; + + const types = { + // Generate an ASCII value. + ascii: function({options}){ + const column = []; + const nb_chars = Math.ceil(random(options) * options.maxWordLength); + for(let i=0; i { // Convert Stream Readable options if underscored - if(options.high_water_mark){ - options.highWaterMark = options.high_water_mark; + if(opts.high_water_mark){ + opts.highWaterMark = opts.high_water_mark; } - if(options.object_mode){ - options.objectMode = options.object_mode; + if(opts.object_mode){ + opts.objectMode = opts.object_mode; } - // Call parent constructor - Stream.Readable.call(this, options); // Clone and camelize options - this.options = {}; - for(const k in options){ - this.options[Generator.camelize(k)] = options[k]; + const options = {}; + for(const k in opts){ + options[camelize(k)] = opts[k]; } // Normalize options const dft = { @@ -5070,110 +5114,100 @@ var csv_generate = (function (exports) { sleep: 0, }; for(const k in dft){ - if(this.options[k] === undefined){ - this.options[k] = dft[k]; + if(options[k] === undefined){ + options[k] = dft[k]; } } // Default values - if(this.options.eof === true){ - this.options.eof = this.options.rowDelimiter; + if(options.eof === true){ + options.eof = options.rowDelimiter; } - // State - this._ = { - start_time: this.options.duration ? Date.now() : null, - fixed_size_buffer: '', - count_written: 0, - count_created: 0, - }; - if(typeof this.options.columns === 'number'){ - this.options.columns = new Array(this.options.columns); + if(typeof options.columns === 'number'){ + options.columns = new Array(options.columns); } - const accepted_header_types = Object.keys(Generator).filter((t) => (!['super_', 'camelize'].includes(t))); - for(let i = 0; i < this.options.columns.length; i++){ - const v = this.options.columns[i] || 'ascii'; + const accepted_header_types = Object.keys(types).filter((t) => (!['super_', 'camelize'].includes(t))); + for(let i = 0; i < options.columns.length; i++){ + const v = options.columns[i] || 'ascii'; if(typeof v === 'string'){ if(!accepted_header_types.includes(v)){ throw Error(`Invalid column type: got "${v}", default values are ${JSON.stringify(accepted_header_types)}`); } - this.options.columns[i] = Generator[v]; + options.columns[i] = types[v]; } } - return this; + return options; }; - util.inherits(Generator, Stream.Readable); - // Generate a random number between 0 and 1 with 2 decimals. The function is idempotent if it detect the "seed" option. - Generator.prototype.random = function(){ - if(this.options.seed){ - return this.options.seed = this.options.seed * Math.PI * 100 % 100 / 100; - }else { - return Math.random(); - } - }; - // Stop the generation. - Generator.prototype.end = function(){ - this.push(null); - }; - // Put new data into the read queue. - Generator.prototype._read = function(size){ + const read = (options, state, size, push, close) => { // Already started const data = []; - let length = this._.fixed_size_buffer.length; + let length = state.fixed_size_buffer.length; if(length !== 0){ - data.push(this._.fixed_size_buffer); + data.push(state.fixed_size_buffer); } // eslint-disable-next-line while(true){ // Time for some rest: flush first and stop later - if((this._.count_created === this.options.length) || (this.options.end && Date.now() > this.options.end) || (this.options.duration && Date.now() > this._.start_time + this.options.duration)){ + if((state.count_created === options.length) || (options.end && Date.now() > options.end) || (options.duration && Date.now() > state.start_time + options.duration)){ // Flush if(data.length){ - if(this.options.objectMode){ + if(options.objectMode){ for(const record of data){ - this.__push(record); + push(record); } }else { - this.__push(data.join('') + (this.options.eof ? this.options.eof : '')); + push(data.join('') + (options.eof ? options.eof : '')); } - this._.end = true; + state.end = true; }else { - this.push(null); + close(); } return; } // Create the record let record = []; let recordLength; - this.options.columns.forEach((fn) => { - record.push(fn(this)); + options.columns.forEach((fn) => { + const result = fn({options: options, state: state}); + const type = typeof result; + if(result !== null && type !== 'string' && type !== 'number'){ + throw Error([ + 'INVALID_VALUE:', + 'values returned by column function must be', + 'a string, a number or null,', + `got ${JSON.stringify(result)}` + ].join(' ')); + } + record.push(result); }); // Obtain record length - if(this.options.objectMode){ + if(options.objectMode){ recordLength = 0; // recordLength is currently equal to the number of columns // This is wrong and shall equal to 1 record only - for(const column of record) + for(const column of record){ recordLength += column.length; + } }else { // Stringify the record - record = (this._.count_created === 0 ? '' : this.options.rowDelimiter)+record.join(this.options.delimiter); + record = (state.count_created === 0 ? '' : options.rowDelimiter)+record.join(options.delimiter); recordLength = record.length; } - this._.count_created++; + state.count_created++; if(length + recordLength > size){ - if(this.options.objectMode){ + if(options.objectMode){ data.push(record); for(const record of data){ - this.__push(record); + push(record); } }else { - if(this.options.fixedSize){ - this._.fixed_size_buffer = record.substr(size - length); + if(options.fixedSize){ + state.fixed_size_buffer = record.substr(size - length); data.push(record.substr(0, size - length)); }else { data.push(record); } - this.__push(data.join('')); + push(data.join('')); } return; } @@ -5181,42 +5215,41 @@ var csv_generate = (function (exports) { data.push(record); } }; + + const Generator = function(options = {}){ + this.options = normalize_options(options); + // Call parent constructor + Stream.Readable.call(this, this.options); + this.state = init_state(this.options); + return this; + }; + util.inherits(Generator, Stream.Readable); + + // Stop the generation. + Generator.prototype.end = function(){ + this.push(null); + }; + // Put new data into the read queue. + Generator.prototype._read = function(size){ + const self = this; + read(this.options, this.state, size, function(chunk) { + self.__push(chunk); + }, function(){ + self.push(null); + }); + }; // Put new data into the read queue. Generator.prototype.__push = function(record){ + // console.log('push', record) const push = () => { - this._.count_written++; + this.state.count_written++; this.push(record); - if(this._.end === true){ + if(this.state.end === true){ return this.push(null); } }; this.options.sleep > 0 ? setTimeout(push, this.options.sleep) : push(); }; - // Generate an ASCII value. - Generator.ascii = function(gen){ - // Column - const column = []; - const nb_chars = Math.ceil(gen.random() * gen.options.maxWordLength); - for(let i=0; i { + // State + return { + start_time: options.duration ? Date.now() : null, + fixed_size_buffer: '', + count_written: 0, + count_created: 0, + }; + }; + + // Generate a random number between 0 and 1 with 2 decimals. The function is idempotent if it detect the "seed" option. + const random = function(options={}){ + if(options.seed){ + return options.seed = options.seed * Math.PI * 100 % 100 / 100; + }else { + return Math.random(); + } + }; + + const types = { + // Generate an ASCII value. + ascii: function({options}){ + const column = []; + const nb_chars = Math.ceil(random(options) * options.maxWordLength); + for(let i=0; i { // Convert Stream Readable options if underscored - if(options.high_water_mark){ - options.highWaterMark = options.high_water_mark; + if(opts.high_water_mark){ + opts.highWaterMark = opts.high_water_mark; } - if(options.object_mode){ - options.objectMode = options.object_mode; + if(opts.object_mode){ + opts.objectMode = opts.object_mode; } - // Call parent constructor - Stream.Readable.call(this, options); // Clone and camelize options - this.options = {}; - for(const k in options){ - this.options[Generator.camelize(k)] = options[k]; + const options = {}; + for(const k in opts){ + options[camelize(k)] = opts[k]; } // Normalize options const dft = { @@ -5070,110 +5114,100 @@ var csv_generate_sync = (function (exports) { sleep: 0, }; for(const k in dft){ - if(this.options[k] === undefined){ - this.options[k] = dft[k]; + if(options[k] === undefined){ + options[k] = dft[k]; } } // Default values - if(this.options.eof === true){ - this.options.eof = this.options.rowDelimiter; + if(options.eof === true){ + options.eof = options.rowDelimiter; } - // State - this._ = { - start_time: this.options.duration ? Date.now() : null, - fixed_size_buffer: '', - count_written: 0, - count_created: 0, - }; - if(typeof this.options.columns === 'number'){ - this.options.columns = new Array(this.options.columns); + if(typeof options.columns === 'number'){ + options.columns = new Array(options.columns); } - const accepted_header_types = Object.keys(Generator).filter((t) => (!['super_', 'camelize'].includes(t))); - for(let i = 0; i < this.options.columns.length; i++){ - const v = this.options.columns[i] || 'ascii'; + const accepted_header_types = Object.keys(types).filter((t) => (!['super_', 'camelize'].includes(t))); + for(let i = 0; i < options.columns.length; i++){ + const v = options.columns[i] || 'ascii'; if(typeof v === 'string'){ if(!accepted_header_types.includes(v)){ throw Error(`Invalid column type: got "${v}", default values are ${JSON.stringify(accepted_header_types)}`); } - this.options.columns[i] = Generator[v]; + options.columns[i] = types[v]; } } - return this; + return options; }; - util.inherits(Generator, Stream.Readable); - // Generate a random number between 0 and 1 with 2 decimals. The function is idempotent if it detect the "seed" option. - Generator.prototype.random = function(){ - if(this.options.seed){ - return this.options.seed = this.options.seed * Math.PI * 100 % 100 / 100; - }else { - return Math.random(); - } - }; - // Stop the generation. - Generator.prototype.end = function(){ - this.push(null); - }; - // Put new data into the read queue. - Generator.prototype._read = function(size){ + const read = (options, state, size, push, close) => { // Already started const data = []; - let length = this._.fixed_size_buffer.length; + let length = state.fixed_size_buffer.length; if(length !== 0){ - data.push(this._.fixed_size_buffer); + data.push(state.fixed_size_buffer); } // eslint-disable-next-line while(true){ // Time for some rest: flush first and stop later - if((this._.count_created === this.options.length) || (this.options.end && Date.now() > this.options.end) || (this.options.duration && Date.now() > this._.start_time + this.options.duration)){ + if((state.count_created === options.length) || (options.end && Date.now() > options.end) || (options.duration && Date.now() > state.start_time + options.duration)){ // Flush if(data.length){ - if(this.options.objectMode){ + if(options.objectMode){ for(const record of data){ - this.__push(record); + push(record); } }else { - this.__push(data.join('') + (this.options.eof ? this.options.eof : '')); + push(data.join('') + (options.eof ? options.eof : '')); } - this._.end = true; + state.end = true; }else { - this.push(null); + close(); } return; } // Create the record let record = []; let recordLength; - this.options.columns.forEach((fn) => { - record.push(fn(this)); + options.columns.forEach((fn) => { + const result = fn({options: options, state: state}); + const type = typeof result; + if(result !== null && type !== 'string' && type !== 'number'){ + throw Error([ + 'INVALID_VALUE:', + 'values returned by column function must be', + 'a string, a number or null,', + `got ${JSON.stringify(result)}` + ].join(' ')); + } + record.push(result); }); // Obtain record length - if(this.options.objectMode){ + if(options.objectMode){ recordLength = 0; // recordLength is currently equal to the number of columns // This is wrong and shall equal to 1 record only - for(const column of record) + for(const column of record){ recordLength += column.length; + } }else { // Stringify the record - record = (this._.count_created === 0 ? '' : this.options.rowDelimiter)+record.join(this.options.delimiter); + record = (state.count_created === 0 ? '' : options.rowDelimiter)+record.join(options.delimiter); recordLength = record.length; } - this._.count_created++; + state.count_created++; if(length + recordLength > size){ - if(this.options.objectMode){ + if(options.objectMode){ data.push(record); for(const record of data){ - this.__push(record); + push(record); } }else { - if(this.options.fixedSize){ - this._.fixed_size_buffer = record.substr(size - length); + if(options.fixedSize){ + state.fixed_size_buffer = record.substr(size - length); data.push(record.substr(0, size - length)); }else { data.push(record); } - this.__push(data.join('')); + push(data.join('')); } return; } @@ -5181,42 +5215,41 @@ var csv_generate_sync = (function (exports) { data.push(record); } }; + + const Generator = function(options = {}){ + this.options = normalize_options(options); + // Call parent constructor + Stream.Readable.call(this, this.options); + this.state = init_state(this.options); + return this; + }; + util.inherits(Generator, Stream.Readable); + + // Stop the generation. + Generator.prototype.end = function(){ + this.push(null); + }; + // Put new data into the read queue. + Generator.prototype._read = function(size){ + const self = this; + read(this.options, this.state, size, function(chunk) { + self.__push(chunk); + }, function(){ + self.push(null); + }); + }; // Put new data into the read queue. Generator.prototype.__push = function(record){ + // console.log('push', record) const push = () => { - this._.count_written++; + this.state.count_written++; this.push(record); - if(this._.end === true){ + if(this.state.end === true){ return this.push(null); } }; this.options.sleep > 0 ? setTimeout(push, this.options.sleep) : push(); }; - // Generate an ASCII value. - Generator.ascii = function(gen){ - // Column - const column = []; - const nb_chars = Math.ceil(gen.random() * gen.options.maxWordLength); - for(let i=0; i { + // State + return { + start_time: options.duration ? Date.now() : null, + fixed_size_buffer: '', + count_written: 0, + count_created: 0, + }; + }; + + // Generate a random number between 0 and 1 with 2 decimals. The function is idempotent if it detect the "seed" option. + const random = function(options={}){ + if(options.seed){ + return options.seed = options.seed * Math.PI * 100 % 100 / 100; + }else { + return Math.random(); + } + }; + + const types = { + // Generate an ASCII value. + ascii: function({options}){ + const column = []; + const nb_chars = Math.ceil(random(options) * options.maxWordLength); + for(let i=0; i { // Convert Stream Readable options if underscored - if(options.high_water_mark){ - options.highWaterMark = options.high_water_mark; + if(opts.high_water_mark){ + opts.highWaterMark = opts.high_water_mark; } - if(options.object_mode){ - options.objectMode = options.object_mode; + if(opts.object_mode){ + opts.objectMode = opts.object_mode; } - // Call parent constructor - Stream.Readable.call(this, options); // Clone and camelize options - this.options = {}; - for(const k in options){ - this.options[Generator.camelize(k)] = options[k]; + const options = {}; + for(const k in opts){ + options[camelize(k)] = opts[k]; } // Normalize options const dft = { @@ -5073,110 +5117,100 @@ sleep: 0, }; for(const k in dft){ - if(this.options[k] === undefined){ - this.options[k] = dft[k]; + if(options[k] === undefined){ + options[k] = dft[k]; } } // Default values - if(this.options.eof === true){ - this.options.eof = this.options.rowDelimiter; + if(options.eof === true){ + options.eof = options.rowDelimiter; } - // State - this._ = { - start_time: this.options.duration ? Date.now() : null, - fixed_size_buffer: '', - count_written: 0, - count_created: 0, - }; - if(typeof this.options.columns === 'number'){ - this.options.columns = new Array(this.options.columns); + if(typeof options.columns === 'number'){ + options.columns = new Array(options.columns); } - const accepted_header_types = Object.keys(Generator).filter((t) => (!['super_', 'camelize'].includes(t))); - for(let i = 0; i < this.options.columns.length; i++){ - const v = this.options.columns[i] || 'ascii'; + const accepted_header_types = Object.keys(types).filter((t) => (!['super_', 'camelize'].includes(t))); + for(let i = 0; i < options.columns.length; i++){ + const v = options.columns[i] || 'ascii'; if(typeof v === 'string'){ if(!accepted_header_types.includes(v)){ throw Error(`Invalid column type: got "${v}", default values are ${JSON.stringify(accepted_header_types)}`); } - this.options.columns[i] = Generator[v]; + options.columns[i] = types[v]; } } - return this; + return options; }; - util.inherits(Generator, Stream.Readable); - // Generate a random number between 0 and 1 with 2 decimals. The function is idempotent if it detect the "seed" option. - Generator.prototype.random = function(){ - if(this.options.seed){ - return this.options.seed = this.options.seed * Math.PI * 100 % 100 / 100; - }else { - return Math.random(); - } - }; - // Stop the generation. - Generator.prototype.end = function(){ - this.push(null); - }; - // Put new data into the read queue. - Generator.prototype._read = function(size){ + const read = (options, state, size, push, close) => { // Already started const data = []; - let length = this._.fixed_size_buffer.length; + let length = state.fixed_size_buffer.length; if(length !== 0){ - data.push(this._.fixed_size_buffer); + data.push(state.fixed_size_buffer); } // eslint-disable-next-line while(true){ // Time for some rest: flush first and stop later - if((this._.count_created === this.options.length) || (this.options.end && Date.now() > this.options.end) || (this.options.duration && Date.now() > this._.start_time + this.options.duration)){ + if((state.count_created === options.length) || (options.end && Date.now() > options.end) || (options.duration && Date.now() > state.start_time + options.duration)){ // Flush if(data.length){ - if(this.options.objectMode){ + if(options.objectMode){ for(const record of data){ - this.__push(record); + push(record); } }else { - this.__push(data.join('') + (this.options.eof ? this.options.eof : '')); + push(data.join('') + (options.eof ? options.eof : '')); } - this._.end = true; + state.end = true; }else { - this.push(null); + close(); } return; } // Create the record let record = []; let recordLength; - this.options.columns.forEach((fn) => { - record.push(fn(this)); + options.columns.forEach((fn) => { + const result = fn({options: options, state: state}); + const type = typeof result; + if(result !== null && type !== 'string' && type !== 'number'){ + throw Error([ + 'INVALID_VALUE:', + 'values returned by column function must be', + 'a string, a number or null,', + `got ${JSON.stringify(result)}` + ].join(' ')); + } + record.push(result); }); // Obtain record length - if(this.options.objectMode){ + if(options.objectMode){ recordLength = 0; // recordLength is currently equal to the number of columns // This is wrong and shall equal to 1 record only - for(const column of record) + for(const column of record){ recordLength += column.length; + } }else { // Stringify the record - record = (this._.count_created === 0 ? '' : this.options.rowDelimiter)+record.join(this.options.delimiter); + record = (state.count_created === 0 ? '' : options.rowDelimiter)+record.join(options.delimiter); recordLength = record.length; } - this._.count_created++; + state.count_created++; if(length + recordLength > size){ - if(this.options.objectMode){ + if(options.objectMode){ data.push(record); for(const record of data){ - this.__push(record); + push(record); } }else { - if(this.options.fixedSize){ - this._.fixed_size_buffer = record.substr(size - length); + if(options.fixedSize){ + state.fixed_size_buffer = record.substr(size - length); data.push(record.substr(0, size - length)); }else { data.push(record); } - this.__push(data.join('')); + push(data.join('')); } return; } @@ -5184,42 +5218,41 @@ data.push(record); } }; + + const Generator = function(options = {}){ + this.options = normalize_options(options); + // Call parent constructor + Stream.Readable.call(this, this.options); + this.state = init_state(this.options); + return this; + }; + util.inherits(Generator, Stream.Readable); + + // Stop the generation. + Generator.prototype.end = function(){ + this.push(null); + }; + // Put new data into the read queue. + Generator.prototype._read = function(size){ + const self = this; + read(this.options, this.state, size, function(chunk) { + self.__push(chunk); + }, function(){ + self.push(null); + }); + }; // Put new data into the read queue. Generator.prototype.__push = function(record){ + // console.log('push', record) const push = () => { - this._.count_written++; + this.state.count_written++; this.push(record); - if(this._.end === true){ + if(this.state.end === true){ return this.push(null); } }; this.options.sleep > 0 ? setTimeout(push, this.options.sleep) : push(); }; - // Generate an ASCII value. - Generator.ascii = function(gen){ - // Column - const column = []; - const nb_chars = Math.ceil(gen.random() * gen.options.maxWordLength); - for(let i=0; i { + // State + return { + start_time: options.duration ? Date.now() : null, + fixed_size_buffer: '', + count_written: 0, + count_created: 0, + }; + }; + + // Generate a random number between 0 and 1 with 2 decimals. The function is idempotent if it detect the "seed" option. + const random = function(options={}){ + if(options.seed){ + return options.seed = options.seed * Math.PI * 100 % 100 / 100; + }else { + return Math.random(); + } + }; + + const types = { + // Generate an ASCII value. + ascii: function({options}){ + const column = []; + const nb_chars = Math.ceil(random(options) * options.maxWordLength); + for(let i=0; i { // Convert Stream Readable options if underscored - if(options.high_water_mark){ - options.highWaterMark = options.high_water_mark; + if(opts.high_water_mark){ + opts.highWaterMark = opts.high_water_mark; } - if(options.object_mode){ - options.objectMode = options.object_mode; + if(opts.object_mode){ + opts.objectMode = opts.object_mode; } - // Call parent constructor - Stream.Readable.call(this, options); // Clone and camelize options - this.options = {}; - for(const k in options){ - this.options[Generator.camelize(k)] = options[k]; + const options = {}; + for(const k in opts){ + options[camelize(k)] = opts[k]; } // Normalize options const dft = { @@ -5073,110 +5117,100 @@ sleep: 0, }; for(const k in dft){ - if(this.options[k] === undefined){ - this.options[k] = dft[k]; + if(options[k] === undefined){ + options[k] = dft[k]; } } // Default values - if(this.options.eof === true){ - this.options.eof = this.options.rowDelimiter; + if(options.eof === true){ + options.eof = options.rowDelimiter; } - // State - this._ = { - start_time: this.options.duration ? Date.now() : null, - fixed_size_buffer: '', - count_written: 0, - count_created: 0, - }; - if(typeof this.options.columns === 'number'){ - this.options.columns = new Array(this.options.columns); + if(typeof options.columns === 'number'){ + options.columns = new Array(options.columns); } - const accepted_header_types = Object.keys(Generator).filter((t) => (!['super_', 'camelize'].includes(t))); - for(let i = 0; i < this.options.columns.length; i++){ - const v = this.options.columns[i] || 'ascii'; + const accepted_header_types = Object.keys(types).filter((t) => (!['super_', 'camelize'].includes(t))); + for(let i = 0; i < options.columns.length; i++){ + const v = options.columns[i] || 'ascii'; if(typeof v === 'string'){ if(!accepted_header_types.includes(v)){ throw Error(`Invalid column type: got "${v}", default values are ${JSON.stringify(accepted_header_types)}`); } - this.options.columns[i] = Generator[v]; + options.columns[i] = types[v]; } } - return this; + return options; }; - util.inherits(Generator, Stream.Readable); - // Generate a random number between 0 and 1 with 2 decimals. The function is idempotent if it detect the "seed" option. - Generator.prototype.random = function(){ - if(this.options.seed){ - return this.options.seed = this.options.seed * Math.PI * 100 % 100 / 100; - }else { - return Math.random(); - } - }; - // Stop the generation. - Generator.prototype.end = function(){ - this.push(null); - }; - // Put new data into the read queue. - Generator.prototype._read = function(size){ + const read = (options, state, size, push, close) => { // Already started const data = []; - let length = this._.fixed_size_buffer.length; + let length = state.fixed_size_buffer.length; if(length !== 0){ - data.push(this._.fixed_size_buffer); + data.push(state.fixed_size_buffer); } // eslint-disable-next-line while(true){ // Time for some rest: flush first and stop later - if((this._.count_created === this.options.length) || (this.options.end && Date.now() > this.options.end) || (this.options.duration && Date.now() > this._.start_time + this.options.duration)){ + if((state.count_created === options.length) || (options.end && Date.now() > options.end) || (options.duration && Date.now() > state.start_time + options.duration)){ // Flush if(data.length){ - if(this.options.objectMode){ + if(options.objectMode){ for(const record of data){ - this.__push(record); + push(record); } }else { - this.__push(data.join('') + (this.options.eof ? this.options.eof : '')); + push(data.join('') + (options.eof ? options.eof : '')); } - this._.end = true; + state.end = true; }else { - this.push(null); + close(); } return; } // Create the record let record = []; let recordLength; - this.options.columns.forEach((fn) => { - record.push(fn(this)); + options.columns.forEach((fn) => { + const result = fn({options: options, state: state}); + const type = typeof result; + if(result !== null && type !== 'string' && type !== 'number'){ + throw Error([ + 'INVALID_VALUE:', + 'values returned by column function must be', + 'a string, a number or null,', + `got ${JSON.stringify(result)}` + ].join(' ')); + } + record.push(result); }); // Obtain record length - if(this.options.objectMode){ + if(options.objectMode){ recordLength = 0; // recordLength is currently equal to the number of columns // This is wrong and shall equal to 1 record only - for(const column of record) + for(const column of record){ recordLength += column.length; + } }else { // Stringify the record - record = (this._.count_created === 0 ? '' : this.options.rowDelimiter)+record.join(this.options.delimiter); + record = (state.count_created === 0 ? '' : options.rowDelimiter)+record.join(options.delimiter); recordLength = record.length; } - this._.count_created++; + state.count_created++; if(length + recordLength > size){ - if(this.options.objectMode){ + if(options.objectMode){ data.push(record); for(const record of data){ - this.__push(record); + push(record); } }else { - if(this.options.fixedSize){ - this._.fixed_size_buffer = record.substr(size - length); + if(options.fixedSize){ + state.fixed_size_buffer = record.substr(size - length); data.push(record.substr(0, size - length)); }else { data.push(record); } - this.__push(data.join('')); + push(data.join('')); } return; } @@ -5184,42 +5218,41 @@ data.push(record); } }; + + const Generator = function(options = {}){ + this.options = normalize_options(options); + // Call parent constructor + Stream.Readable.call(this, this.options); + this.state = init_state(this.options); + return this; + }; + util.inherits(Generator, Stream.Readable); + + // Stop the generation. + Generator.prototype.end = function(){ + this.push(null); + }; + // Put new data into the read queue. + Generator.prototype._read = function(size){ + const self = this; + read(this.options, this.state, size, function(chunk) { + self.__push(chunk); + }, function(){ + self.push(null); + }); + }; // Put new data into the read queue. Generator.prototype.__push = function(record){ + // console.log('push', record) const push = () => { - this._.count_written++; + this.state.count_written++; this.push(record); - if(this._.end === true){ + if(this.state.end === true){ return this.push(null); } }; this.options.sleep > 0 ? setTimeout(push, this.options.sleep) : push(); }; - // Generate an ASCII value. - Generator.ascii = function(gen){ - // Column - const column = []; - const nb_chars = Math.ceil(gen.random() * gen.options.maxWordLength); - for(let i=0; i { + // State + return { + start_time: options.duration ? Date.now() : null, + fixed_size_buffer: '', + count_written: 0, + count_created: 0, + }; +}; + +export {init_state}; diff --git a/packages/csv-generate/lib/api/normalize_options.js b/packages/csv-generate/lib/api/normalize_options.js new file mode 100644 index 000000000..072611629 --- /dev/null +++ b/packages/csv-generate/lib/api/normalize_options.js @@ -0,0 +1,63 @@ + +import {types} from './types.js'; + +const camelize = function(str){ + return str.replace(/_([a-z])/gi, function(_, match){ + return match.toUpperCase(); + }); +}; + +const normalize_options = (opts) => { + // Convert Stream Readable options if underscored + if(opts.high_water_mark){ + opts.highWaterMark = opts.high_water_mark; + } + if(opts.object_mode){ + opts.objectMode = opts.object_mode; + } + // Clone and camelize options + const options = {}; + for(const k in opts){ + options[camelize(k)] = opts[k]; + } + // Normalize options + const dft = { + columns: 8, + delimiter: ',', + duration: null, + encoding: null, + end: null, + eof: false, + fixedSize: false, + length: -1, + maxWordLength: 16, + rowDelimiter: '\n', + seed: false, + sleep: 0, + }; + for(const k in dft){ + if(options[k] === undefined){ + options[k] = dft[k]; + } + } + // Default values + if(options.eof === true){ + options.eof = options.rowDelimiter; + } + if(typeof options.columns === 'number'){ + options.columns = new Array(options.columns); + } + const accepted_header_types = Object.keys(types).filter((t) => (!['super_', 'camelize'].includes(t))); + for(let i = 0; i < options.columns.length; i++){ + const v = options.columns[i] || 'ascii'; + if(typeof v === 'string'){ + if(!accepted_header_types.includes(v)){ + throw Error(`Invalid column type: got "${v}", default values are ${JSON.stringify(accepted_header_types)}`); + } + options.columns[i] = types[v]; + } + } + return options; +}; + +export {normalize_options}; diff --git a/packages/csv-generate/lib/api/random.js b/packages/csv-generate/lib/api/random.js new file mode 100644 index 000000000..04692e558 --- /dev/null +++ b/packages/csv-generate/lib/api/random.js @@ -0,0 +1,11 @@ + +// Generate a random number between 0 and 1 with 2 decimals. The function is idempotent if it detect the "seed" option. +const random = function(options={}){ + if(options.seed){ + return options.seed = options.seed * Math.PI * 100 % 100 / 100; + }else{ + return Math.random(); + } +}; + +export {random}; diff --git a/packages/csv-generate/lib/api/read.js b/packages/csv-generate/lib/api/read.js new file mode 100644 index 000000000..e4191a3bb --- /dev/null +++ b/packages/csv-generate/lib/api/read.js @@ -0,0 +1,81 @@ + + +const read = (options, state, size, push, close) => { + // Already started + const data = []; + let length = state.fixed_size_buffer.length; + if(length !== 0){ + data.push(state.fixed_size_buffer); + } + // eslint-disable-next-line + while(true){ + // Time for some rest: flush first and stop later + if((state.count_created === options.length) || (options.end && Date.now() > options.end) || (options.duration && Date.now() > state.start_time + options.duration)){ + // Flush + if(data.length){ + if(options.objectMode){ + for(const record of data){ + push(record); + } + }else{ + push(data.join('') + (options.eof ? options.eof : '')); + } + state.end = true; + }else{ + close(); + } + return; + } + // Create the record + let record = []; + let recordLength; + options.columns.forEach((fn) => { + const result = fn({options: options, state: state}); + const type = typeof result; + if(result !== null && type !== 'string' && type !== 'number'){ + throw Error([ + 'INVALID_VALUE:', + 'values returned by column function must be', + 'a string, a number or null,', + `got ${JSON.stringify(result)}` + ].join(' ')); + } + record.push(result); + }); + // Obtain record length + if(options.objectMode){ + recordLength = 0; + // recordLength is currently equal to the number of columns + // This is wrong and shall equal to 1 record only + for(const column of record){ + recordLength += column.length; + } + }else{ + // Stringify the record + record = (state.count_created === 0 ? '' : options.rowDelimiter)+record.join(options.delimiter); + recordLength = record.length; + } + state.count_created++; + if(length + recordLength > size){ + if(options.objectMode){ + data.push(record); + for(const record of data){ + push(record); + } + }else{ + if(options.fixedSize){ + state.fixed_size_buffer = record.substr(size - length); + data.push(record.substr(0, size - length)); + }else{ + data.push(record); + } + push(data.join('')); + } + return; + } + length += recordLength; + data.push(record); + } +}; + +export {read}; diff --git a/packages/csv-generate/lib/api/types.js b/packages/csv-generate/lib/api/types.js new file mode 100644 index 000000000..e2e703b11 --- /dev/null +++ b/packages/csv-generate/lib/api/types.js @@ -0,0 +1,25 @@ + +import {random} from './random.js'; + +const types = { + // Generate an ASCII value. + ascii: function({options}){ + const column = []; + const nb_chars = Math.ceil(random(options) * options.maxWordLength); + for(let i=0; i (!['super_', 'camelize'].includes(t))); - for(let i = 0; i < this.options.columns.length; i++){ - const v = this.options.columns[i] || 'ascii'; - if(typeof v === 'string'){ - if(!accepted_header_types.includes(v)){ - throw Error(`Invalid column type: got "${v}", default values are ${JSON.stringify(accepted_header_types)}`); - } - this.options.columns[i] = Generator[v]; - } - } + stream.Readable.call(this, this.options); + this.state = init_state(this.options); return this; }; util.inherits(Generator, stream.Readable); -// Generate a random number between 0 and 1 with 2 decimals. The function is idempotent if it detect the "seed" option. -Generator.prototype.random = function(){ - if(this.options.seed){ - return this.options.seed = this.options.seed * Math.PI * 100 % 100 / 100; - }else{ - return Math.random(); - } -}; // Stop the generation. Generator.prototype.end = function(){ this.push(null); }; // Put new data into the read queue. Generator.prototype._read = function(size){ - // Already started - const data = []; - let length = this._.fixed_size_buffer.length; - if(length !== 0){ - data.push(this._.fixed_size_buffer); - } - // eslint-disable-next-line - while(true){ - // Time for some rest: flush first and stop later - if((this._.count_created === this.options.length) || (this.options.end && Date.now() > this.options.end) || (this.options.duration && Date.now() > this._.start_time + this.options.duration)){ - // Flush - if(data.length){ - if(this.options.objectMode){ - for(const record of data){ - this.__push(record); - } - }else{ - this.__push(data.join('') + (this.options.eof ? this.options.eof : '')); - } - this._.end = true; - }else{ - this.push(null); - } - return; - } - // Create the record - let record = []; - let recordLength; - this.options.columns.forEach((fn) => { - record.push(fn(this)); - }); - // Obtain record length - if(this.options.objectMode){ - recordLength = 0; - // recordLength is currently equal to the number of columns - // This is wrong and shall equal to 1 record only - for(const column of record) - recordLength += column.length; - }else{ - // Stringify the record - record = (this._.count_created === 0 ? '' : this.options.rowDelimiter)+record.join(this.options.delimiter); - recordLength = record.length; - } - this._.count_created++; - if(length + recordLength > size){ - if(this.options.objectMode){ - data.push(record); - for(const record of data){ - this.__push(record); - } - }else{ - if(this.options.fixedSize){ - this._.fixed_size_buffer = record.substr(size - length); - data.push(record.substr(0, size - length)); - }else{ - data.push(record); - } - this.__push(data.join('')); - } - return; - } - length += recordLength; - data.push(record); - } + const self = this; + read(this.options, this.state, size, function(chunk) { + self.__push(chunk); + }, function(){ + self.push(null); + }); }; // Put new data into the read queue. Generator.prototype.__push = function(record){ + // console.log('push', record) const push = () => { - this._.count_written++; + this.state.count_written++; this.push(record); - if(this._.end === true){ + if(this.state.end === true){ return this.push(null); } }; this.options.sleep > 0 ? setTimeout(push, this.options.sleep) : push(); }; -// Generate an ASCII value. -Generator.ascii = function(gen){ - // Column - const column = []; - const nb_chars = Math.ceil(gen.random() * gen.options.maxWordLength); - for(let i=0; i { + const options = normalize_options(opts || {}); + const state = init_state(options); + return new ReadableStream({ + async pull(controller) { + read(options, state, 1024, function(chunk) { + chunk = Buffer.from(chunk); + controller.enqueue(chunk) + }, function(){ + controller.close() + }); + } + }, {highWaterMark: 1024}); + // return new Generator(options || {}) +} + +export {generate}; diff --git a/packages/csv-generate/package.json b/packages/csv-generate/package.json index 8795720dc..5ea6fbfb7 100644 --- a/packages/csv-generate/package.json +++ b/packages/csv-generate/package.json @@ -36,6 +36,10 @@ "import": "./lib/sync.js", "require": "./dist/cjs/sync.cjs" }, + "./stream": { + "import": "./lib/stream.js", + "require": "./dist/cjs/stream.cjs" + }, "./browser/esm": "./dist/esm/index.js", "./browser/esm/sync": "./dist/esm/sync.js" }, diff --git a/packages/csv-generate/test/api.web_stream.coffee b/packages/csv-generate/test/api.web_stream.coffee new file mode 100644 index 000000000..6bace88da --- /dev/null +++ b/packages/csv-generate/test/api.web_stream.coffee @@ -0,0 +1,52 @@ + +import {generate as generateStream} from '../lib/stream.js' +import {generate as generateClassic} from '../lib/index.js' + +describe 'api stream', -> + + it.skip 'perf classic', -> + console.time('classic') + generator = generateClassic({ + objectMode: true, + length: 10000000 + }); + for await record from generator + continue + console.timeEnd('classic') + + it.skip 'perf stream', -> + console.time('stream') + generator = generateStream({ + objectMode: true, + length: 10000000 + }); + reader = generator.getReader() + while true + { done, value } = await reader.read() + break if done + # for await chunk from generator.getReader().read() + # console.log(Buffer.from(chunk).toString()); + console.timeEnd('stream') + + it 'perf stream with iterator', -> + generator = generateStream({ + objectMode: true, + length: 10 + }); + records = [] + for await record from generator + records.push record + records.length.should.eql 10 + + it 'perf stream with reader', -> + generator = generateStream({ + objectMode: true, + length: 10 + }); + records = [] + reader = generator.getReader() + while true + { done, record } = await reader.read() + break if done + records.push record + records.length.should.eql 10 diff --git a/packages/csv-generate/test/options.columns.coffee b/packages/csv-generate/test/options.columns.coffee index 73fb38f88..b7dac5618 100644 --- a/packages/csv-generate/test/options.columns.coffee +++ b/packages/csv-generate/test/options.columns.coffee @@ -43,23 +43,72 @@ describe 'option columns', -> catch err err.message.should.eql 'Invalid column type: got "invalid", default values are ["ascii","int","bool"]' next() + + describe 'user function', -> - it 'as user function', (next) -> - @timeout 1000000 - count = 0 - data = [] - generator = generate columns: [ - -> 'a' - -> 'b' - ] - generator.on 'readable', -> - while d = generator.read() - data.push d - if count++ is 2 - generator.end() - generator.on 'error', next - generator.on 'end', -> - data - .join('').split('\n')[1].split(',') - .should.eql ['a', 'b'] - next() + it 'accept string or null or number', (next) -> + @timeout 1000000 + data = [] + generator = generate length: 1, columns: [ + -> 'a' + -> null + -> 1 + ] + generator.on 'readable', -> + while d = generator.read() + data.push d.toString() + generator.on 'error', next + generator.on 'end', -> + data.should.eql ['a,,1'] + next() + + it 'validate return argument', (next) -> + @timeout 1000000 + generator = generate objectMode: true, columns: [ + -> {} + ] + generator.on 'readable', -> + while d = generator.read() then true + generator.on 'error', (err) -> + err.message.should.eql [ + 'INVALID_VALUE:' + 'values returned by column function must be' + 'a string, a number or null,' + 'got {}' + ].join ' ' + next() + generator.on 'end', -> next Error('Oh no') + + it 'validate arguments in objectMode', (next) -> + @timeout 1000000 + data = [] + generator = generate objectMode: true, length: 1, columns: [ + ({options}) -> JSON.stringify(options) + ({state}) -> JSON.stringify(state) + ] + generator.on 'readable', -> + while d = generator.read() + data.push JSON.parse(d[0]), JSON.parse(d[1]) + generator.on 'error', next + generator.on 'end', -> + data.should.eql [ + objectMode: true + length: 1 + columns: [ null, null ] + delimiter: ',' + duration: null + encoding: null + end: null + eof: false + fixedSize: false + maxWordLength: 16 + rowDelimiter: '\n' + seed: false + sleep: 0 + , + start_time: null + fixed_size_buffer: '' + count_written: 0 + count_created: 0 + ] + next() diff --git a/packages/csv-generate/test/options.seed.coffee b/packages/csv-generate/test/options.seed.coffee index 7f0b003b3..0d6753702 100644 --- a/packages/csv-generate/test/options.seed.coffee +++ b/packages/csv-generate/test/options.seed.coffee @@ -1,25 +1,26 @@ import { generate } from '../lib/index.js' +import { random } from '../lib/api/random.js' describe 'option seed', -> describe 'without seed', -> it 'generate different values', -> - generate().random().should.not.equal generate().random() + random(generate().options).should.not.equal random(generate().options) it 'generate between 0 and 1', -> - generate().random().should.be.above 0 - generate().random().should.be.below 1 + random(generate().options).should.be.above 0 + random(generate().options).should.be.below 1 describe 'with seed', -> it 'generate same values', -> - generate(seed: 1).random().should.equal generate(seed: 1).random() + random(generate(seed: 1).options).should.equal random(generate(seed: 1).options) it 'generate between 0 and 1', -> - generate(seed: 1).random().should.be.above 0 - generate(seed: 1).random().should.be.below 1 + random(generate(seed: 1).options).should.be.above 0 + random(generate(seed: 1).options).should.be.below 1 it 'generate data with highWaterMark', (next) -> @timeout 1000000 diff --git a/packages/csv-parse/dist/cjs/index.cjs b/packages/csv-parse/dist/cjs/index.cjs index 871e1aac6..429644eba 100644 --- a/packages/csv-parse/dist/cjs/index.cjs +++ b/packages/csv-parse/dist/cjs/index.cjs @@ -4,6 +4,55 @@ Object.defineProperty(exports, '__esModule', { value: true }); var stream = require('stream'); +const is_object = function(obj){ + return (typeof obj === 'object' && obj !== null && !Array.isArray(obj)); +}; + +class CsvError extends Error { + constructor(code, message, options, ...contexts) { + if(Array.isArray(message)) message = message.join(' '); + super(message); + if(Error.captureStackTrace !== undefined){ + Error.captureStackTrace(this, CsvError); + } + this.code = code; + for(const context of contexts){ + for(const key in context){ + const value = context[key]; + this[key] = Buffer.isBuffer(value) ? value.toString(options.encoding) : value == null ? value : JSON.parse(JSON.stringify(value)); + } + } + } +} + +const normalize_columns_array = function(columns){ + const normalizedColumns = []; + for(let i = 0, l = columns.length; i < l; i++){ + const column = columns[i]; + if(column === undefined || column === null || column === false){ + normalizedColumns[i] = { disabled: true }; + }else if(typeof column === 'string'){ + normalizedColumns[i] = { name: column }; + }else if(is_object(column)){ + if(typeof column.name !== 'string'){ + throw new CsvError('CSV_OPTION_COLUMNS_MISSING_NAME', [ + 'Option columns missing name:', + `property "name" is required at position ${i}`, + 'when column is an object literal' + ]); + } + normalizedColumns[i] = column; + }else { + throw new CsvError('CSV_INVALID_COLUMN_DEFINITION', [ + 'Invalid column definition:', + 'expect a string or a literal object,', + `got ${JSON.stringify(column)} at position ${i}` + ]); + } + } + return normalizedColumns; +}; + class ResizeableBuffer{ constructor(size=100){ this.size = size; @@ -66,1210 +115,1192 @@ class ResizeableBuffer{ } } -// white space characters -// https://en.wikipedia.org/wiki/Whitespace_character -// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions/Character_Classes#Types -// \f\n\r\t\v\u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff -const tab = 9; -const nl = 10; // \n, 0x0A in hexadecimal, 10 in decimal -const np = 12; -const cr = 13; // \r, 0x0D in hexadécimal, 13 in decimal -const space = 32; -const boms = { - // Note, the following are equals: - // Buffer.from("\ufeff") - // Buffer.from([239, 187, 191]) - // Buffer.from('EFBBBF', 'hex') - 'utf8': Buffer.from([239, 187, 191]), - // Note, the following are equals: - // Buffer.from "\ufeff", 'utf16le - // Buffer.from([255, 254]) - 'utf16le': Buffer.from([255, 254]) +const init_state = function(options){ + return { + bomSkipped: false, + bufBytesStart: 0, + castField: options.cast_function, + commenting: false, + // Current error encountered by a record + error: undefined, + enabled: options.from_line === 1, + escaping: false, + escapeIsQuote: Buffer.isBuffer(options.escape) && Buffer.isBuffer(options.quote) && Buffer.compare(options.escape, options.quote) === 0, + // columns can be `false`, `true`, `Array` + expectedRecordLength: Array.isArray(options.columns) ? options.columns.length : undefined, + field: new ResizeableBuffer(20), + firstLineToHeaders: options.cast_first_line_to_header, + needMoreDataSize: Math.max( + // Skip if the remaining buffer smaller than comment + options.comment !== null ? options.comment.length : 0, + // Skip if the remaining buffer can be delimiter + ...options.delimiter.map((delimiter) => delimiter.length), + // Skip if the remaining buffer can be escape sequence + options.quote !== null ? options.quote.length : 0, + ), + previousBuf: undefined, + quoting: false, + stop: false, + rawBuffer: new ResizeableBuffer(100), + record: [], + recordHasError: false, + record_length: 0, + recordDelimiterMaxLength: options.record_delimiter.length === 0 ? 2 : Math.max(...options.record_delimiter.map((v) => v.length)), + trimChars: [Buffer.from(' ', options.encoding)[0], Buffer.from('\t', options.encoding)[0]], + wasQuoting: false, + wasRowDelimiter: false + }; }; -class CsvError extends Error { - constructor(code, message, options, ...contexts) { - if(Array.isArray(message)) message = message.join(' '); - super(message); - if(Error.captureStackTrace !== undefined){ - Error.captureStackTrace(this, CsvError); - } - this.code = code; - for(const context of contexts){ - for(const key in context){ - const value = context[key]; - this[key] = Buffer.isBuffer(value) ? value.toString(options.encoding) : value == null ? value : JSON.parse(JSON.stringify(value)); - } - } - } -} - const underscore = function(str){ return str.replace(/([A-Z])/g, function(_, match){ return '_' + match.toLowerCase(); }); }; -const isObject = function(obj){ - return (typeof obj === 'object' && obj !== null && !Array.isArray(obj)); -}; - -const isRecordEmpty = function(record){ - return record.every((field) => field == null || field.toString && field.toString().trim() === ''); -}; - -const normalizeColumnsArray = function(columns){ - const normalizedColumns = []; - for(let i = 0, l = columns.length; i < l; i++){ - const column = columns[i]; - if(column === undefined || column === null || column === false){ - normalizedColumns[i] = { disabled: true }; - }else if(typeof column === 'string'){ - normalizedColumns[i] = { name: column }; - }else if(isObject(column)){ - if(typeof column.name !== 'string'){ - throw new CsvError('CSV_OPTION_COLUMNS_MISSING_NAME', [ - 'Option columns missing name:', - `property "name" is required at position ${i}`, - 'when column is an object literal' - ]); - } - normalizedColumns[i] = column; - }else { - throw new CsvError('CSV_INVALID_COLUMN_DEFINITION', [ - 'Invalid column definition:', - 'expect a string or a literal object,', - `got ${JSON.stringify(column)} at position ${i}` - ]); - } +const normalize_options = function(opts){ + const options = {}; + // Merge with user options + for(const opt in opts){ + options[underscore(opt)] = opts[opt]; } - return normalizedColumns; -}; - -class Parser extends stream.Transform { - constructor(opts = {}){ - super({...{readableObjectMode: true}, ...opts, encoding: null}); - this.__originalOptions = opts; - this.__normalizeOptions(opts); + // Normalize option `encoding` + // Note: defined first because other options depends on it + // to convert chars/strings into buffers. + if(options.encoding === undefined || options.encoding === true){ + options.encoding = 'utf8'; + }else if(options.encoding === null || options.encoding === false){ + options.encoding = null; + }else if(typeof options.encoding !== 'string' && options.encoding !== null){ + throw new CsvError('CSV_INVALID_OPTION_ENCODING', [ + 'Invalid option encoding:', + 'encoding must be a string or null to return a buffer,', + `got ${JSON.stringify(options.encoding)}` + ], options); } - __normalizeOptions(opts){ - const options = {}; - // Merge with user options - for(const opt in opts){ - options[underscore(opt)] = opts[opt]; - } - // Normalize option `encoding` - // Note: defined first because other options depends on it - // to convert chars/strings into buffers. - if(options.encoding === undefined || options.encoding === true){ - options.encoding = 'utf8'; - }else if(options.encoding === null || options.encoding === false){ - options.encoding = null; - }else if(typeof options.encoding !== 'string' && options.encoding !== null){ - throw new CsvError('CSV_INVALID_OPTION_ENCODING', [ - 'Invalid option encoding:', - 'encoding must be a string or null to return a buffer,', - `got ${JSON.stringify(options.encoding)}` - ], options); - } - // Normalize option `bom` - if(options.bom === undefined || options.bom === null || options.bom === false){ - options.bom = false; - }else if(options.bom !== true){ - throw new CsvError('CSV_INVALID_OPTION_BOM', [ - 'Invalid option bom:', 'bom must be true,', - `got ${JSON.stringify(options.bom)}` - ], options); - } - // Normalize option `cast` - let fnCastField = null; - if(options.cast === undefined || options.cast === null || options.cast === false || options.cast === ''){ - options.cast = undefined; - }else if(typeof options.cast === 'function'){ - fnCastField = options.cast; - options.cast = true; - }else if(options.cast !== true){ - throw new CsvError('CSV_INVALID_OPTION_CAST', [ - 'Invalid option cast:', 'cast must be true or a function,', - `got ${JSON.stringify(options.cast)}` - ], options); - } - // Normalize option `cast_date` - if(options.cast_date === undefined || options.cast_date === null || options.cast_date === false || options.cast_date === ''){ - options.cast_date = false; - }else if(options.cast_date === true){ - options.cast_date = function(value){ - const date = Date.parse(value); - return !isNaN(date) ? new Date(date) : value; - }; - }else { - throw new CsvError('CSV_INVALID_OPTION_CAST_DATE', [ - 'Invalid option cast_date:', 'cast_date must be true or a function,', - `got ${JSON.stringify(options.cast_date)}` - ], options); - } - // Normalize option `columns` - let fnFirstLineToHeaders = null; - if(options.columns === true){ - // Fields in the first line are converted as-is to columns - fnFirstLineToHeaders = undefined; - }else if(typeof options.columns === 'function'){ - fnFirstLineToHeaders = options.columns; - options.columns = true; - }else if(Array.isArray(options.columns)){ - options.columns = normalizeColumnsArray(options.columns); - }else if(options.columns === undefined || options.columns === null || options.columns === false){ - options.columns = false; - }else { - throw new CsvError('CSV_INVALID_OPTION_COLUMNS', [ - 'Invalid option columns:', - 'expect an array, a function or true,', - `got ${JSON.stringify(options.columns)}` + // Normalize option `bom` + if(options.bom === undefined || options.bom === null || options.bom === false){ + options.bom = false; + }else if(options.bom !== true){ + throw new CsvError('CSV_INVALID_OPTION_BOM', [ + 'Invalid option bom:', 'bom must be true,', + `got ${JSON.stringify(options.bom)}` + ], options); + } + // Normalize option `cast` + options.cast_function = null; + if(options.cast === undefined || options.cast === null || options.cast === false || options.cast === ''){ + options.cast = undefined; + }else if(typeof options.cast === 'function'){ + options.cast_function = options.cast; + options.cast = true; + }else if(options.cast !== true){ + throw new CsvError('CSV_INVALID_OPTION_CAST', [ + 'Invalid option cast:', 'cast must be true or a function,', + `got ${JSON.stringify(options.cast)}` + ], options); + } + // Normalize option `cast_date` + if(options.cast_date === undefined || options.cast_date === null || options.cast_date === false || options.cast_date === ''){ + options.cast_date = false; + }else if(options.cast_date === true){ + options.cast_date = function(value){ + const date = Date.parse(value); + return !isNaN(date) ? new Date(date) : value; + }; + }else { + throw new CsvError('CSV_INVALID_OPTION_CAST_DATE', [ + 'Invalid option cast_date:', 'cast_date must be true or a function,', + `got ${JSON.stringify(options.cast_date)}` + ], options); + } + // Normalize option `columns` + options.cast_first_line_to_header = null; + if(options.columns === true){ + // Fields in the first line are converted as-is to columns + options.cast_first_line_to_header = undefined; + }else if(typeof options.columns === 'function'){ + options.cast_first_line_to_header = options.columns; + options.columns = true; + }else if(Array.isArray(options.columns)){ + options.columns = normalize_columns_array(options.columns); + }else if(options.columns === undefined || options.columns === null || options.columns === false){ + options.columns = false; + }else { + throw new CsvError('CSV_INVALID_OPTION_COLUMNS', [ + 'Invalid option columns:', + 'expect an array, a function or true,', + `got ${JSON.stringify(options.columns)}` + ], options); + } + // Normalize option `group_columns_by_name` + if(options.group_columns_by_name === undefined || options.group_columns_by_name === null || options.group_columns_by_name === false){ + options.group_columns_by_name = false; + }else if(options.group_columns_by_name !== true){ + throw new CsvError('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ + 'Invalid option group_columns_by_name:', + 'expect an boolean,', + `got ${JSON.stringify(options.group_columns_by_name)}` + ], options); + }else if(options.columns === false){ + throw new CsvError('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ + 'Invalid option group_columns_by_name:', + 'the `columns` mode must be activated.' + ], options); + } + // Normalize option `comment` + if(options.comment === undefined || options.comment === null || options.comment === false || options.comment === ''){ + options.comment = null; + }else { + if(typeof options.comment === 'string'){ + options.comment = Buffer.from(options.comment, options.encoding); + } + if(!Buffer.isBuffer(options.comment)){ + throw new CsvError('CSV_INVALID_OPTION_COMMENT', [ + 'Invalid option comment:', + 'comment must be a buffer or a string,', + `got ${JSON.stringify(options.comment)}` ], options); } - // Normalize option `group_columns_by_name` - if(options.group_columns_by_name === undefined || options.group_columns_by_name === null || options.group_columns_by_name === false){ - options.group_columns_by_name = false; - }else if(options.group_columns_by_name !== true){ - throw new CsvError('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ - 'Invalid option group_columns_by_name:', - 'expect an boolean,', - `got ${JSON.stringify(options.group_columns_by_name)}` - ], options); - }else if(options.columns === false){ - throw new CsvError('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ - 'Invalid option group_columns_by_name:', - 'the `columns` mode must be activated.' - ], options); + } + // Normalize option `delimiter` + const delimiter_json = JSON.stringify(options.delimiter); + if(!Array.isArray(options.delimiter)) options.delimiter = [options.delimiter]; + if(options.delimiter.length === 0){ + throw new CsvError('CSV_INVALID_OPTION_DELIMITER', [ + 'Invalid option delimiter:', + 'delimiter must be a non empty string or buffer or array of string|buffer,', + `got ${delimiter_json}` + ], options); + } + options.delimiter = options.delimiter.map(function(delimiter){ + if(delimiter === undefined || delimiter === null || delimiter === false){ + return Buffer.from(',', options.encoding); } - // Normalize option `comment` - if(options.comment === undefined || options.comment === null || options.comment === false || options.comment === ''){ - options.comment = null; - }else { - if(typeof options.comment === 'string'){ - options.comment = Buffer.from(options.comment, options.encoding); - } - if(!Buffer.isBuffer(options.comment)){ - throw new CsvError('CSV_INVALID_OPTION_COMMENT', [ - 'Invalid option comment:', - 'comment must be a buffer or a string,', - `got ${JSON.stringify(options.comment)}` - ], options); - } + if(typeof delimiter === 'string'){ + delimiter = Buffer.from(delimiter, options.encoding); } - // Normalize option `delimiter` - const delimiter_json = JSON.stringify(options.delimiter); - if(!Array.isArray(options.delimiter)) options.delimiter = [options.delimiter]; - if(options.delimiter.length === 0){ + if(!Buffer.isBuffer(delimiter) || delimiter.length === 0){ throw new CsvError('CSV_INVALID_OPTION_DELIMITER', [ 'Invalid option delimiter:', 'delimiter must be a non empty string or buffer or array of string|buffer,', `got ${delimiter_json}` ], options); } - options.delimiter = options.delimiter.map(function(delimiter){ - if(delimiter === undefined || delimiter === null || delimiter === false){ - return Buffer.from(',', options.encoding); - } - if(typeof delimiter === 'string'){ - delimiter = Buffer.from(delimiter, options.encoding); - } - if(!Buffer.isBuffer(delimiter) || delimiter.length === 0){ - throw new CsvError('CSV_INVALID_OPTION_DELIMITER', [ - 'Invalid option delimiter:', - 'delimiter must be a non empty string or buffer or array of string|buffer,', - `got ${delimiter_json}` - ], options); - } - return delimiter; - }); - // Normalize option `escape` - if(options.escape === undefined || options.escape === true){ - options.escape = Buffer.from('"', options.encoding); - }else if(typeof options.escape === 'string'){ - options.escape = Buffer.from(options.escape, options.encoding); - }else if (options.escape === null || options.escape === false){ - options.escape = null; - } - if(options.escape !== null){ - if(!Buffer.isBuffer(options.escape)){ - throw new Error(`Invalid Option: escape must be a buffer, a string or a boolean, got ${JSON.stringify(options.escape)}`); - } + return delimiter; + }); + // Normalize option `escape` + if(options.escape === undefined || options.escape === true){ + options.escape = Buffer.from('"', options.encoding); + }else if(typeof options.escape === 'string'){ + options.escape = Buffer.from(options.escape, options.encoding); + }else if (options.escape === null || options.escape === false){ + options.escape = null; + } + if(options.escape !== null){ + if(!Buffer.isBuffer(options.escape)){ + throw new Error(`Invalid Option: escape must be a buffer, a string or a boolean, got ${JSON.stringify(options.escape)}`); } - // Normalize option `from` - if(options.from === undefined || options.from === null){ - options.from = 1; - }else { - if(typeof options.from === 'string' && /\d+/.test(options.from)){ - options.from = parseInt(options.from); - } - if(Number.isInteger(options.from)){ - if(options.from < 0){ - throw new Error(`Invalid Option: from must be a positive integer, got ${JSON.stringify(opts.from)}`); - } - }else { - throw new Error(`Invalid Option: from must be an integer, got ${JSON.stringify(options.from)}`); + } + // Normalize option `from` + if(options.from === undefined || options.from === null){ + options.from = 1; + }else { + if(typeof options.from === 'string' && /\d+/.test(options.from)){ + options.from = parseInt(options.from); + } + if(Number.isInteger(options.from)){ + if(options.from < 0){ + throw new Error(`Invalid Option: from must be a positive integer, got ${JSON.stringify(opts.from)}`); } - } - // Normalize option `from_line` - if(options.from_line === undefined || options.from_line === null){ - options.from_line = 1; }else { - if(typeof options.from_line === 'string' && /\d+/.test(options.from_line)){ - options.from_line = parseInt(options.from_line); - } - if(Number.isInteger(options.from_line)){ - if(options.from_line <= 0){ - throw new Error(`Invalid Option: from_line must be a positive integer greater than 0, got ${JSON.stringify(opts.from_line)}`); - } - }else { - throw new Error(`Invalid Option: from_line must be an integer, got ${JSON.stringify(opts.from_line)}`); - } + throw new Error(`Invalid Option: from must be an integer, got ${JSON.stringify(options.from)}`); } - // Normalize options `ignore_last_delimiters` - if(options.ignore_last_delimiters === undefined || options.ignore_last_delimiters === null){ - options.ignore_last_delimiters = false; - }else if(typeof options.ignore_last_delimiters === 'number'){ - options.ignore_last_delimiters = Math.floor(options.ignore_last_delimiters); - if(options.ignore_last_delimiters === 0){ - options.ignore_last_delimiters = false; + } + // Normalize option `from_line` + if(options.from_line === undefined || options.from_line === null){ + options.from_line = 1; + }else { + if(typeof options.from_line === 'string' && /\d+/.test(options.from_line)){ + options.from_line = parseInt(options.from_line); + } + if(Number.isInteger(options.from_line)){ + if(options.from_line <= 0){ + throw new Error(`Invalid Option: from_line must be a positive integer greater than 0, got ${JSON.stringify(opts.from_line)}`); } - }else if(typeof options.ignore_last_delimiters !== 'boolean'){ - throw new CsvError('CSV_INVALID_OPTION_IGNORE_LAST_DELIMITERS', [ - 'Invalid option `ignore_last_delimiters`:', - 'the value must be a boolean value or an integer,', - `got ${JSON.stringify(options.ignore_last_delimiters)}` - ], options); - } - if(options.ignore_last_delimiters === true && options.columns === false){ - throw new CsvError('CSV_IGNORE_LAST_DELIMITERS_REQUIRES_COLUMNS', [ - 'The option `ignore_last_delimiters`', - 'requires the activation of the `columns` option' - ], options); - } - // Normalize option `info` - if(options.info === undefined || options.info === null || options.info === false){ - options.info = false; - }else if(options.info !== true){ - throw new Error(`Invalid Option: info must be true, got ${JSON.stringify(options.info)}`); - } - // Normalize option `max_record_size` - if(options.max_record_size === undefined || options.max_record_size === null || options.max_record_size === false){ - options.max_record_size = 0; - }else if(Number.isInteger(options.max_record_size) && options.max_record_size >= 0);else if(typeof options.max_record_size === 'string' && /\d+/.test(options.max_record_size)){ - options.max_record_size = parseInt(options.max_record_size); }else { - throw new Error(`Invalid Option: max_record_size must be a positive integer, got ${JSON.stringify(options.max_record_size)}`); - } - // Normalize option `objname` - if(options.objname === undefined || options.objname === null || options.objname === false){ - options.objname = undefined; - }else if(Buffer.isBuffer(options.objname)){ - if(options.objname.length === 0){ - throw new Error(`Invalid Option: objname must be a non empty buffer`); - } - if(options.encoding === null);else { - options.objname = options.objname.toString(options.encoding); - } - }else if(typeof options.objname === 'string'){ - if(options.objname.length === 0){ - throw new Error(`Invalid Option: objname must be a non empty string`); - } - // Great, nothing to do - }else if(typeof options.objname === 'number');else { - throw new Error(`Invalid Option: objname must be a string or a buffer, got ${options.objname}`); - } - if(options.objname !== undefined){ - if(typeof options.objname === 'number'){ - if(options.columns !== false){ - throw Error('Invalid Option: objname index cannot be combined with columns or be defined as a field'); - } - }else { // A string or a buffer - if(options.columns === false){ - throw Error('Invalid Option: objname field must be combined with columns or be defined as an index'); - } - } + throw new Error(`Invalid Option: from_line must be an integer, got ${JSON.stringify(opts.from_line)}`); } - // Normalize option `on_record` - if(options.on_record === undefined || options.on_record === null){ - options.on_record = undefined; - }else if(typeof options.on_record !== 'function'){ - throw new CsvError('CSV_INVALID_OPTION_ON_RECORD', [ - 'Invalid option `on_record`:', - 'expect a function,', - `got ${JSON.stringify(options.on_record)}` - ], options); + } + // Normalize options `ignore_last_delimiters` + if(options.ignore_last_delimiters === undefined || options.ignore_last_delimiters === null){ + options.ignore_last_delimiters = false; + }else if(typeof options.ignore_last_delimiters === 'number'){ + options.ignore_last_delimiters = Math.floor(options.ignore_last_delimiters); + if(options.ignore_last_delimiters === 0){ + options.ignore_last_delimiters = false; } - // Normalize option `quote` - if(options.quote === null || options.quote === false || options.quote === ''){ - options.quote = null; - }else { - if(options.quote === undefined || options.quote === true){ - options.quote = Buffer.from('"', options.encoding); - }else if(typeof options.quote === 'string'){ - options.quote = Buffer.from(options.quote, options.encoding); + }else if(typeof options.ignore_last_delimiters !== 'boolean'){ + throw new CsvError('CSV_INVALID_OPTION_IGNORE_LAST_DELIMITERS', [ + 'Invalid option `ignore_last_delimiters`:', + 'the value must be a boolean value or an integer,', + `got ${JSON.stringify(options.ignore_last_delimiters)}` + ], options); + } + if(options.ignore_last_delimiters === true && options.columns === false){ + throw new CsvError('CSV_IGNORE_LAST_DELIMITERS_REQUIRES_COLUMNS', [ + 'The option `ignore_last_delimiters`', + 'requires the activation of the `columns` option' + ], options); + } + // Normalize option `info` + if(options.info === undefined || options.info === null || options.info === false){ + options.info = false; + }else if(options.info !== true){ + throw new Error(`Invalid Option: info must be true, got ${JSON.stringify(options.info)}`); + } + // Normalize option `max_record_size` + if(options.max_record_size === undefined || options.max_record_size === null || options.max_record_size === false){ + options.max_record_size = 0; + }else if(Number.isInteger(options.max_record_size) && options.max_record_size >= 0);else if(typeof options.max_record_size === 'string' && /\d+/.test(options.max_record_size)){ + options.max_record_size = parseInt(options.max_record_size); + }else { + throw new Error(`Invalid Option: max_record_size must be a positive integer, got ${JSON.stringify(options.max_record_size)}`); + } + // Normalize option `objname` + if(options.objname === undefined || options.objname === null || options.objname === false){ + options.objname = undefined; + }else if(Buffer.isBuffer(options.objname)){ + if(options.objname.length === 0){ + throw new Error(`Invalid Option: objname must be a non empty buffer`); + } + if(options.encoding === null);else { + options.objname = options.objname.toString(options.encoding); + } + }else if(typeof options.objname === 'string'){ + if(options.objname.length === 0){ + throw new Error(`Invalid Option: objname must be a non empty string`); + } + // Great, nothing to do + }else if(typeof options.objname === 'number');else { + throw new Error(`Invalid Option: objname must be a string or a buffer, got ${options.objname}`); + } + if(options.objname !== undefined){ + if(typeof options.objname === 'number'){ + if(options.columns !== false){ + throw Error('Invalid Option: objname index cannot be combined with columns or be defined as a field'); } - if(!Buffer.isBuffer(options.quote)){ - throw new Error(`Invalid Option: quote must be a buffer or a string, got ${JSON.stringify(options.quote)}`); + }else { // A string or a buffer + if(options.columns === false){ + throw Error('Invalid Option: objname field must be combined with columns or be defined as an index'); } } - // Normalize option `raw` - if(options.raw === undefined || options.raw === null || options.raw === false){ - options.raw = false; - }else if(options.raw !== true){ - throw new Error(`Invalid Option: raw must be true, got ${JSON.stringify(options.raw)}`); + } + // Normalize option `on_record` + if(options.on_record === undefined || options.on_record === null){ + options.on_record = undefined; + }else if(typeof options.on_record !== 'function'){ + throw new CsvError('CSV_INVALID_OPTION_ON_RECORD', [ + 'Invalid option `on_record`:', + 'expect a function,', + `got ${JSON.stringify(options.on_record)}` + ], options); + } + // Normalize option `quote` + if(options.quote === null || options.quote === false || options.quote === ''){ + options.quote = null; + }else { + if(options.quote === undefined || options.quote === true){ + options.quote = Buffer.from('"', options.encoding); + }else if(typeof options.quote === 'string'){ + options.quote = Buffer.from(options.quote, options.encoding); + } + if(!Buffer.isBuffer(options.quote)){ + throw new Error(`Invalid Option: quote must be a buffer or a string, got ${JSON.stringify(options.quote)}`); } - // Normalize option `record_delimiter` - if(options.record_delimiter === undefined){ - options.record_delimiter = []; - }else if(typeof options.record_delimiter === 'string' || Buffer.isBuffer(options.record_delimiter)){ - if(options.record_delimiter.length === 0){ - throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ - 'Invalid option `record_delimiter`:', - 'value must be a non empty string or buffer,', - `got ${JSON.stringify(options.record_delimiter)}` - ], options); - } - options.record_delimiter = [options.record_delimiter]; - }else if(!Array.isArray(options.record_delimiter)){ + } + // Normalize option `raw` + if(options.raw === undefined || options.raw === null || options.raw === false){ + options.raw = false; + }else if(options.raw !== true){ + throw new Error(`Invalid Option: raw must be true, got ${JSON.stringify(options.raw)}`); + } + // Normalize option `record_delimiter` + if(options.record_delimiter === undefined){ + options.record_delimiter = []; + }else if(typeof options.record_delimiter === 'string' || Buffer.isBuffer(options.record_delimiter)){ + if(options.record_delimiter.length === 0){ throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ 'Invalid option `record_delimiter`:', - 'value must be a string, a buffer or array of string|buffer,', + 'value must be a non empty string or buffer,', `got ${JSON.stringify(options.record_delimiter)}` ], options); } - options.record_delimiter = options.record_delimiter.map(function(rd, i){ - if(typeof rd !== 'string' && ! Buffer.isBuffer(rd)){ - throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ - 'Invalid option `record_delimiter`:', - 'value must be a string, a buffer or array of string|buffer', - `at index ${i},`, - `got ${JSON.stringify(rd)}` - ], options); - }else if(rd.length === 0){ - throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ - 'Invalid option `record_delimiter`:', - 'value must be a non empty string or buffer', - `at index ${i},`, - `got ${JSON.stringify(rd)}` - ], options); - } - if(typeof rd === 'string'){ - rd = Buffer.from(rd, options.encoding); - } - return rd; - }); - // Normalize option `relax_column_count` - if(typeof options.relax_column_count === 'boolean');else if(options.relax_column_count === undefined || options.relax_column_count === null){ - options.relax_column_count = false; - }else { - throw new Error(`Invalid Option: relax_column_count must be a boolean, got ${JSON.stringify(options.relax_column_count)}`); - } - if(typeof options.relax_column_count_less === 'boolean');else if(options.relax_column_count_less === undefined || options.relax_column_count_less === null){ - options.relax_column_count_less = false; - }else { - throw new Error(`Invalid Option: relax_column_count_less must be a boolean, got ${JSON.stringify(options.relax_column_count_less)}`); - } - if(typeof options.relax_column_count_more === 'boolean');else if(options.relax_column_count_more === undefined || options.relax_column_count_more === null){ - options.relax_column_count_more = false; - }else { - throw new Error(`Invalid Option: relax_column_count_more must be a boolean, got ${JSON.stringify(options.relax_column_count_more)}`); - } - // Normalize option `relax_quotes` - if(typeof options.relax_quotes === 'boolean');else if(options.relax_quotes === undefined || options.relax_quotes === null){ - options.relax_quotes = false; - }else { - throw new Error(`Invalid Option: relax_quotes must be a boolean, got ${JSON.stringify(options.relax_quotes)}`); - } - // Normalize option `skip_empty_lines` - if(typeof options.skip_empty_lines === 'boolean');else if(options.skip_empty_lines === undefined || options.skip_empty_lines === null){ - options.skip_empty_lines = false; - }else { - throw new Error(`Invalid Option: skip_empty_lines must be a boolean, got ${JSON.stringify(options.skip_empty_lines)}`); - } - // Normalize option `skip_records_with_empty_values` - if(typeof options.skip_records_with_empty_values === 'boolean');else if(options.skip_records_with_empty_values === undefined || options.skip_records_with_empty_values === null){ - options.skip_records_with_empty_values = false; - }else { - throw new Error(`Invalid Option: skip_records_with_empty_values must be a boolean, got ${JSON.stringify(options.skip_records_with_empty_values)}`); - } - // Normalize option `skip_records_with_error` - if(typeof options.skip_records_with_error === 'boolean');else if(options.skip_records_with_error === undefined || options.skip_records_with_error === null){ - options.skip_records_with_error = false; - }else { - throw new Error(`Invalid Option: skip_records_with_error must be a boolean, got ${JSON.stringify(options.skip_records_with_error)}`); - } - // Normalize option `rtrim` - if(options.rtrim === undefined || options.rtrim === null || options.rtrim === false){ - options.rtrim = false; - }else if(options.rtrim !== true){ - throw new Error(`Invalid Option: rtrim must be a boolean, got ${JSON.stringify(options.rtrim)}`); - } - // Normalize option `ltrim` - if(options.ltrim === undefined || options.ltrim === null || options.ltrim === false){ - options.ltrim = false; - }else if(options.ltrim !== true){ - throw new Error(`Invalid Option: ltrim must be a boolean, got ${JSON.stringify(options.ltrim)}`); - } - // Normalize option `trim` - if(options.trim === undefined || options.trim === null || options.trim === false){ - options.trim = false; - }else if(options.trim !== true){ - throw new Error(`Invalid Option: trim must be a boolean, got ${JSON.stringify(options.trim)}`); - } - // Normalize options `trim`, `ltrim` and `rtrim` - if(options.trim === true && opts.ltrim !== false){ - options.ltrim = true; - }else if(options.ltrim !== true){ - options.ltrim = false; + options.record_delimiter = [options.record_delimiter]; + }else if(!Array.isArray(options.record_delimiter)){ + throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ + 'Invalid option `record_delimiter`:', + 'value must be a string, a buffer or array of string|buffer,', + `got ${JSON.stringify(options.record_delimiter)}` + ], options); + } + options.record_delimiter = options.record_delimiter.map(function(rd, i){ + if(typeof rd !== 'string' && ! Buffer.isBuffer(rd)){ + throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ + 'Invalid option `record_delimiter`:', + 'value must be a string, a buffer or array of string|buffer', + `at index ${i},`, + `got ${JSON.stringify(rd)}` + ], options); + }else if(rd.length === 0){ + throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ + 'Invalid option `record_delimiter`:', + 'value must be a non empty string or buffer', + `at index ${i},`, + `got ${JSON.stringify(rd)}` + ], options); } - if(options.trim === true && opts.rtrim !== false){ - options.rtrim = true; - }else if(options.rtrim !== true){ - options.rtrim = false; + if(typeof rd === 'string'){ + rd = Buffer.from(rd, options.encoding); } - // Normalize option `to` - if(options.to === undefined || options.to === null){ - options.to = -1; - }else { - if(typeof options.to === 'string' && /\d+/.test(options.to)){ - options.to = parseInt(options.to); - } - if(Number.isInteger(options.to)){ - if(options.to <= 0){ - throw new Error(`Invalid Option: to must be a positive integer greater than 0, got ${JSON.stringify(opts.to)}`); - } - }else { - throw new Error(`Invalid Option: to must be an integer, got ${JSON.stringify(opts.to)}`); + return rd; + }); + // Normalize option `relax_column_count` + if(typeof options.relax_column_count === 'boolean');else if(options.relax_column_count === undefined || options.relax_column_count === null){ + options.relax_column_count = false; + }else { + throw new Error(`Invalid Option: relax_column_count must be a boolean, got ${JSON.stringify(options.relax_column_count)}`); + } + if(typeof options.relax_column_count_less === 'boolean');else if(options.relax_column_count_less === undefined || options.relax_column_count_less === null){ + options.relax_column_count_less = false; + }else { + throw new Error(`Invalid Option: relax_column_count_less must be a boolean, got ${JSON.stringify(options.relax_column_count_less)}`); + } + if(typeof options.relax_column_count_more === 'boolean');else if(options.relax_column_count_more === undefined || options.relax_column_count_more === null){ + options.relax_column_count_more = false; + }else { + throw new Error(`Invalid Option: relax_column_count_more must be a boolean, got ${JSON.stringify(options.relax_column_count_more)}`); + } + // Normalize option `relax_quotes` + if(typeof options.relax_quotes === 'boolean');else if(options.relax_quotes === undefined || options.relax_quotes === null){ + options.relax_quotes = false; + }else { + throw new Error(`Invalid Option: relax_quotes must be a boolean, got ${JSON.stringify(options.relax_quotes)}`); + } + // Normalize option `skip_empty_lines` + if(typeof options.skip_empty_lines === 'boolean');else if(options.skip_empty_lines === undefined || options.skip_empty_lines === null){ + options.skip_empty_lines = false; + }else { + throw new Error(`Invalid Option: skip_empty_lines must be a boolean, got ${JSON.stringify(options.skip_empty_lines)}`); + } + // Normalize option `skip_records_with_empty_values` + if(typeof options.skip_records_with_empty_values === 'boolean');else if(options.skip_records_with_empty_values === undefined || options.skip_records_with_empty_values === null){ + options.skip_records_with_empty_values = false; + }else { + throw new Error(`Invalid Option: skip_records_with_empty_values must be a boolean, got ${JSON.stringify(options.skip_records_with_empty_values)}`); + } + // Normalize option `skip_records_with_error` + if(typeof options.skip_records_with_error === 'boolean');else if(options.skip_records_with_error === undefined || options.skip_records_with_error === null){ + options.skip_records_with_error = false; + }else { + throw new Error(`Invalid Option: skip_records_with_error must be a boolean, got ${JSON.stringify(options.skip_records_with_error)}`); + } + // Normalize option `rtrim` + if(options.rtrim === undefined || options.rtrim === null || options.rtrim === false){ + options.rtrim = false; + }else if(options.rtrim !== true){ + throw new Error(`Invalid Option: rtrim must be a boolean, got ${JSON.stringify(options.rtrim)}`); + } + // Normalize option `ltrim` + if(options.ltrim === undefined || options.ltrim === null || options.ltrim === false){ + options.ltrim = false; + }else if(options.ltrim !== true){ + throw new Error(`Invalid Option: ltrim must be a boolean, got ${JSON.stringify(options.ltrim)}`); + } + // Normalize option `trim` + if(options.trim === undefined || options.trim === null || options.trim === false){ + options.trim = false; + }else if(options.trim !== true){ + throw new Error(`Invalid Option: trim must be a boolean, got ${JSON.stringify(options.trim)}`); + } + // Normalize options `trim`, `ltrim` and `rtrim` + if(options.trim === true && opts.ltrim !== false){ + options.ltrim = true; + }else if(options.ltrim !== true){ + options.ltrim = false; + } + if(options.trim === true && opts.rtrim !== false){ + options.rtrim = true; + }else if(options.rtrim !== true){ + options.rtrim = false; + } + // Normalize option `to` + if(options.to === undefined || options.to === null){ + options.to = -1; + }else { + if(typeof options.to === 'string' && /\d+/.test(options.to)){ + options.to = parseInt(options.to); + } + if(Number.isInteger(options.to)){ + if(options.to <= 0){ + throw new Error(`Invalid Option: to must be a positive integer greater than 0, got ${JSON.stringify(opts.to)}`); } - } - // Normalize option `to_line` - if(options.to_line === undefined || options.to_line === null){ - options.to_line = -1; }else { - if(typeof options.to_line === 'string' && /\d+/.test(options.to_line)){ - options.to_line = parseInt(options.to_line); - } - if(Number.isInteger(options.to_line)){ - if(options.to_line <= 0){ - throw new Error(`Invalid Option: to_line must be a positive integer greater than 0, got ${JSON.stringify(opts.to_line)}`); - } - }else { - throw new Error(`Invalid Option: to_line must be an integer, got ${JSON.stringify(opts.to_line)}`); - } + throw new Error(`Invalid Option: to must be an integer, got ${JSON.stringify(opts.to)}`); } - this.info = { - bytes: 0, - comment_lines: 0, - empty_lines: 0, - invalid_field_length: 0, - lines: 1, - records: 0 - }; - this.options = options; - this.state = { - bomSkipped: false, - bufBytesStart: 0, - castField: fnCastField, - commenting: false, - // Current error encountered by a record - error: undefined, - enabled: options.from_line === 1, - escaping: false, - escapeIsQuote: Buffer.isBuffer(options.escape) && Buffer.isBuffer(options.quote) && Buffer.compare(options.escape, options.quote) === 0, - // columns can be `false`, `true`, `Array` - expectedRecordLength: Array.isArray(options.columns) ? options.columns.length : undefined, - field: new ResizeableBuffer(20), - firstLineToHeaders: fnFirstLineToHeaders, - needMoreDataSize: Math.max( - // Skip if the remaining buffer smaller than comment - options.comment !== null ? options.comment.length : 0, - // Skip if the remaining buffer can be delimiter - ...options.delimiter.map((delimiter) => delimiter.length), - // Skip if the remaining buffer can be escape sequence - options.quote !== null ? options.quote.length : 0, - ), - previousBuf: undefined, - quoting: false, - stop: false, - rawBuffer: new ResizeableBuffer(100), - record: [], - recordHasError: false, - record_length: 0, - recordDelimiterMaxLength: options.record_delimiter.length === 0 ? 2 : Math.max(...options.record_delimiter.map((v) => v.length)), - trimChars: [Buffer.from(' ', options.encoding)[0], Buffer.from('\t', options.encoding)[0]], - wasQuoting: false, - wasRowDelimiter: false - }; } - // Implementation of `Transform._transform` - _transform(buf, encoding, callback){ - if(this.state.stop === true){ - return; - } - const err = this.__parse(buf, false); - if(err !== undefined){ - this.state.stop = true; + // Normalize option `to_line` + if(options.to_line === undefined || options.to_line === null){ + options.to_line = -1; + }else { + if(typeof options.to_line === 'string' && /\d+/.test(options.to_line)){ + options.to_line = parseInt(options.to_line); + } + if(Number.isInteger(options.to_line)){ + if(options.to_line <= 0){ + throw new Error(`Invalid Option: to_line must be a positive integer greater than 0, got ${JSON.stringify(opts.to_line)}`); + } + }else { + throw new Error(`Invalid Option: to_line must be an integer, got ${JSON.stringify(opts.to_line)}`); } - callback(err); } - // Implementation of `Transform._flush` - _flush(callback){ - if(this.state.stop === true){ - return; - } - const err = this.__parse(undefined, true); - callback(err); - } - // Central parser implementation - __parse(nextBuf, end){ - const {bom, comment, escape, from_line, ltrim, max_record_size, quote, raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options; - let {record_delimiter} = this.options; - const {bomSkipped, previousBuf, rawBuffer, escapeIsQuote} = this.state; - let buf; - if(previousBuf === undefined){ - if(nextBuf === undefined){ - // Handle empty string - this.push(null); - return; - }else { - buf = nextBuf; - } - }else if(previousBuf !== undefined && nextBuf === undefined){ - buf = previousBuf; - }else { - buf = Buffer.concat([previousBuf, nextBuf]); - } - // Handle UTF BOM - if(bomSkipped === false){ - if(bom === false){ - this.state.bomSkipped = true; - }else if(buf.length < 3){ - // No enough data - if(end === false){ - // Wait for more data - this.state.previousBuf = buf; + return options; +}; + +const isRecordEmpty = function(record){ + return record.every((field) => field == null || field.toString && field.toString().trim() === ''); +}; + +// white space characters +// https://en.wikipedia.org/wiki/Whitespace_character +// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions/Character_Classes#Types +// \f\n\r\t\v\u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff +const tab = 9; +const nl = 10; // \n, 0x0A in hexadecimal, 10 in decimal +const np = 12; +const cr = 13; // \r, 0x0D in hexadécimal, 13 in decimal +const space = 32; +const boms = { + // Note, the following are equals: + // Buffer.from("\ufeff") + // Buffer.from([239, 187, 191]) + // Buffer.from('EFBBBF', 'hex') + 'utf8': Buffer.from([239, 187, 191]), + // Note, the following are equals: + // Buffer.from "\ufeff", 'utf16le + // Buffer.from([255, 254]) + 'utf16le': Buffer.from([255, 254]) +}; + +const transform = function(original_options = {}) { + const info = { + bytes: 0, + comment_lines: 0, + empty_lines: 0, + invalid_field_length: 0, + lines: 1, + records: 0 + }; + const options = normalize_options(original_options); + return { + info: info, + original_options: original_options, + options: options, + state: init_state(options), + __needMoreData: function(i, bufLen, end){ + if(end) return false; + const {quote} = this.options; + const {quoting, needMoreDataSize, recordDelimiterMaxLength} = this.state; + const numOfCharLeft = bufLen - i - 1; + const requiredLength = Math.max( + needMoreDataSize, + // Skip if the remaining buffer smaller than record delimiter + recordDelimiterMaxLength, + // Skip if the remaining buffer can be record delimiter following the closing quote + // 1 is for quote.length + quoting ? (quote.length + recordDelimiterMaxLength) : 0, + ); + return numOfCharLeft < requiredLength; + }, + // Central parser implementation + parse: function(nextBuf, end, push, close){ + const {bom, comment, escape, from_line, ltrim, max_record_size, quote, raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options; + let {record_delimiter} = this.options; + const {bomSkipped, previousBuf, rawBuffer, escapeIsQuote} = this.state; + let buf; + if(previousBuf === undefined){ + if(nextBuf === undefined){ + // Handle empty string + close(); return; + }else { + buf = nextBuf; } + }else if(previousBuf !== undefined && nextBuf === undefined){ + buf = previousBuf; }else { - for(const encoding in boms){ - if(boms[encoding].compare(buf, 0, boms[encoding].length) === 0){ - // Skip BOM - const bomLength = boms[encoding].length; - this.state.bufBytesStart += bomLength; - buf = buf.slice(bomLength); - // Renormalize original options with the new encoding - this.__normalizeOptions({...this.__originalOptions, encoding: encoding}); - break; + buf = Buffer.concat([previousBuf, nextBuf]); + } + // Handle UTF BOM + if(bomSkipped === false){ + if(bom === false){ + this.state.bomSkipped = true; + }else if(buf.length < 3){ + // No enough data + if(end === false){ + // Wait for more data + this.state.previousBuf = buf; + return; + } + }else { + for(const encoding in boms){ + if(boms[encoding].compare(buf, 0, boms[encoding].length) === 0){ + // Skip BOM + const bomLength = boms[encoding].length; + this.state.bufBytesStart += bomLength; + buf = buf.slice(bomLength); + // Renormalize original options with the new encoding + this.options = normalize_options({...this.original_options, encoding: encoding}); + break; + } } + this.state.bomSkipped = true; } - this.state.bomSkipped = true; } - } - const bufLen = buf.length; - let pos; - for(pos = 0; pos < bufLen; pos++){ - // Ensure we get enough space to look ahead - // There should be a way to move this out of the loop - if(this.__needMoreData(pos, bufLen, end)){ - break; - } - if(this.state.wasRowDelimiter === true){ - this.info.lines++; - this.state.wasRowDelimiter = false; - } - if(to_line !== -1 && this.info.lines > to_line){ - this.state.stop = true; - this.push(null); - return; - } - // Auto discovery of record_delimiter, unix, mac and windows supported - if(this.state.quoting === false && record_delimiter.length === 0){ - const record_delimiterCount = this.__autoDiscoverRecordDelimiter(buf, pos); - if(record_delimiterCount){ - record_delimiter = this.options.record_delimiter; + const bufLen = buf.length; + let pos; + for(pos = 0; pos < bufLen; pos++){ + // Ensure we get enough space to look ahead + // There should be a way to move this out of the loop + if(this.__needMoreData(pos, bufLen, end)){ + break; } - } - const chr = buf[pos]; - if(raw === true){ - rawBuffer.append(chr); - } - if((chr === cr || chr === nl) && this.state.wasRowDelimiter === false){ - this.state.wasRowDelimiter = true; - } - // Previous char was a valid escape char - // treat the current char as a regular char - if(this.state.escaping === true){ - this.state.escaping = false; - }else { - // Escape is only active inside quoted fields - // We are quoting, the char is an escape chr and there is a chr to escape - // if(escape !== null && this.state.quoting === true && chr === escape && pos + 1 < bufLen){ - if(escape !== null && this.state.quoting === true && this.__isEscape(buf, pos, chr) && pos + escape.length < bufLen){ - if(escapeIsQuote){ - if(this.__isQuote(buf, pos+escape.length)){ - this.state.escaping = true; - pos += escape.length - 1; - continue; - } - }else { - this.state.escaping = true; - pos += escape.length - 1; - continue; + if(this.state.wasRowDelimiter === true){ + this.info.lines++; + this.state.wasRowDelimiter = false; + } + if(to_line !== -1 && this.info.lines > to_line){ + this.state.stop = true; + close(); + return; + } + // Auto discovery of record_delimiter, unix, mac and windows supported + if(this.state.quoting === false && record_delimiter.length === 0){ + const record_delimiterCount = this.__autoDiscoverRecordDelimiter(buf, pos); + if(record_delimiterCount){ + record_delimiter = this.options.record_delimiter; } } - // Not currently escaping and chr is a quote - // TODO: need to compare bytes instead of single char - if(this.state.commenting === false && this.__isQuote(buf, pos)){ - if(this.state.quoting === true){ - const nextChr = buf[pos+quote.length]; - const isNextChrTrimable = rtrim && this.__isCharTrimable(nextChr); - const isNextChrComment = comment !== null && this.__compareBytes(comment, buf, pos+quote.length, nextChr); - const isNextChrDelimiter = this.__isDelimiter(buf, pos+quote.length, nextChr); - const isNextChrRecordDelimiter = record_delimiter.length === 0 ? this.__autoDiscoverRecordDelimiter(buf, pos+quote.length) : this.__isRecordDelimiter(nextChr, buf, pos+quote.length); - // Escape a quote - // Treat next char as a regular character - if(escape !== null && this.__isEscape(buf, pos, chr) && this.__isQuote(buf, pos + escape.length)){ + const chr = buf[pos]; + if(raw === true){ + rawBuffer.append(chr); + } + if((chr === cr || chr === nl) && this.state.wasRowDelimiter === false){ + this.state.wasRowDelimiter = true; + } + // Previous char was a valid escape char + // treat the current char as a regular char + if(this.state.escaping === true){ + this.state.escaping = false; + }else { + // Escape is only active inside quoted fields + // We are quoting, the char is an escape chr and there is a chr to escape + // if(escape !== null && this.state.quoting === true && chr === escape && pos + 1 < bufLen){ + if(escape !== null && this.state.quoting === true && this.__isEscape(buf, pos, chr) && pos + escape.length < bufLen){ + if(escapeIsQuote){ + if(this.__isQuote(buf, pos+escape.length)){ + this.state.escaping = true; + pos += escape.length - 1; + continue; + } + }else { + this.state.escaping = true; pos += escape.length - 1; - }else if(!nextChr || isNextChrDelimiter || isNextChrRecordDelimiter || isNextChrComment || isNextChrTrimable){ - this.state.quoting = false; - this.state.wasQuoting = true; - pos += quote.length - 1; continue; - }else if(relax_quotes === false){ - const err = this.__error( - new CsvError('CSV_INVALID_CLOSING_QUOTE', [ - 'Invalid Closing Quote:', - `got "${String.fromCharCode(nextChr)}"`, - `at line ${this.info.lines}`, - 'instead of delimiter, record delimiter, trimable character', - '(if activated) or comment', - ], this.options, this.__infoField()) - ); - if(err !== undefined) return err; - }else { - this.state.quoting = false; - this.state.wasQuoting = true; - this.state.field.prepend(quote); - pos += quote.length - 1; } - }else { - if(this.state.field.length !== 0){ - // In relax_quotes mode, treat opening quote preceded by chrs as regular - if(relax_quotes === false){ + } + // Not currently escaping and chr is a quote + // TODO: need to compare bytes instead of single char + if(this.state.commenting === false && this.__isQuote(buf, pos)){ + if(this.state.quoting === true){ + const nextChr = buf[pos+quote.length]; + const isNextChrTrimable = rtrim && this.__isCharTrimable(nextChr); + const isNextChrComment = comment !== null && this.__compareBytes(comment, buf, pos+quote.length, nextChr); + const isNextChrDelimiter = this.__isDelimiter(buf, pos+quote.length, nextChr); + const isNextChrRecordDelimiter = record_delimiter.length === 0 ? this.__autoDiscoverRecordDelimiter(buf, pos+quote.length) : this.__isRecordDelimiter(nextChr, buf, pos+quote.length); + // Escape a quote + // Treat next char as a regular character + if(escape !== null && this.__isEscape(buf, pos, chr) && this.__isQuote(buf, pos + escape.length)){ + pos += escape.length - 1; + }else if(!nextChr || isNextChrDelimiter || isNextChrRecordDelimiter || isNextChrComment || isNextChrTrimable){ + this.state.quoting = false; + this.state.wasQuoting = true; + pos += quote.length - 1; + continue; + }else if(relax_quotes === false){ const err = this.__error( - new CsvError('INVALID_OPENING_QUOTE', [ - 'Invalid Opening Quote:', - `a quote is found inside a field at line ${this.info.lines}`, - ], this.options, this.__infoField(), { - field: this.state.field, - }) + new CsvError('CSV_INVALID_CLOSING_QUOTE', [ + 'Invalid Closing Quote:', + `got "${String.fromCharCode(nextChr)}"`, + `at line ${this.info.lines}`, + 'instead of delimiter, record delimiter, trimable character', + '(if activated) or comment', + ], this.options, this.__infoField()) ); if(err !== undefined) return err; + }else { + this.state.quoting = false; + this.state.wasQuoting = true; + this.state.field.prepend(quote); + pos += quote.length - 1; } }else { - this.state.quoting = true; - pos += quote.length - 1; - continue; - } - } - } - if(this.state.quoting === false){ - const recordDelimiterLength = this.__isRecordDelimiter(chr, buf, pos); - if(recordDelimiterLength !== 0){ - // Do not emit comments which take a full line - const skipCommentLine = this.state.commenting && (this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0); - if(skipCommentLine){ - this.info.comment_lines++; - // Skip full comment line - }else { - // Activate records emition if above from_line - if(this.state.enabled === false && this.info.lines + (this.state.wasRowDelimiter === true ? 1: 0) >= from_line){ - this.state.enabled = true; - this.__resetField(); - this.__resetRecord(); - pos += recordDelimiterLength - 1; + if(this.state.field.length !== 0){ + // In relax_quotes mode, treat opening quote preceded by chrs as regular + if(relax_quotes === false){ + const err = this.__error( + new CsvError('INVALID_OPENING_QUOTE', [ + 'Invalid Opening Quote:', + `a quote is found inside a field at line ${this.info.lines}`, + ], this.options, this.__infoField(), { + field: this.state.field, + }) + ); + if(err !== undefined) return err; + } + }else { + this.state.quoting = true; + pos += quote.length - 1; continue; } - // Skip if line is empty and skip_empty_lines activated - if(skip_empty_lines === true && this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0){ - this.info.empty_lines++; - pos += recordDelimiterLength - 1; - continue; + } + } + if(this.state.quoting === false){ + const recordDelimiterLength = this.__isRecordDelimiter(chr, buf, pos); + if(recordDelimiterLength !== 0){ + // Do not emit comments which take a full line + const skipCommentLine = this.state.commenting && (this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0); + if(skipCommentLine){ + this.info.comment_lines++; + // Skip full comment line + }else { + // Activate records emition if above from_line + if(this.state.enabled === false && this.info.lines + (this.state.wasRowDelimiter === true ? 1: 0) >= from_line){ + this.state.enabled = true; + this.__resetField(); + this.__resetRecord(); + pos += recordDelimiterLength - 1; + continue; + } + // Skip if line is empty and skip_empty_lines activated + if(skip_empty_lines === true && this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0){ + this.info.empty_lines++; + pos += recordDelimiterLength - 1; + continue; + } + this.info.bytes = this.state.bufBytesStart + pos; + const errField = this.__onField(); + if(errField !== undefined) return errField; + this.info.bytes = this.state.bufBytesStart + pos + recordDelimiterLength; + const errRecord = this.__onRecord(push); + if(errRecord !== undefined) return errRecord; + if(to !== -1 && this.info.records >= to){ + this.state.stop = true; + close(); + return; + } } + this.state.commenting = false; + pos += recordDelimiterLength - 1; + continue; + } + if(this.state.commenting){ + continue; + } + const commentCount = comment === null ? 0 : this.__compareBytes(comment, buf, pos, chr); + if(commentCount !== 0){ + this.state.commenting = true; + continue; + } + const delimiterLength = this.__isDelimiter(buf, pos, chr); + if(delimiterLength !== 0){ this.info.bytes = this.state.bufBytesStart + pos; const errField = this.__onField(); if(errField !== undefined) return errField; - this.info.bytes = this.state.bufBytesStart + pos + recordDelimiterLength; - const errRecord = this.__onRecord(); - if(errRecord !== undefined) return errRecord; - if(to !== -1 && this.info.records >= to){ - this.state.stop = true; - this.push(null); - return; - } + pos += delimiterLength - 1; + continue; } - this.state.commenting = false; - pos += recordDelimiterLength - 1; - continue; - } - if(this.state.commenting){ - continue; } - const commentCount = comment === null ? 0 : this.__compareBytes(comment, buf, pos, chr); - if(commentCount !== 0){ - this.state.commenting = true; - continue; - } - const delimiterLength = this.__isDelimiter(buf, pos, chr); - if(delimiterLength !== 0){ - this.info.bytes = this.state.bufBytesStart + pos; - const errField = this.__onField(); - if(errField !== undefined) return errField; - pos += delimiterLength - 1; - continue; + } + if(this.state.commenting === false){ + if(max_record_size !== 0 && this.state.record_length + this.state.field.length > max_record_size){ + const err = this.__error( + new CsvError('CSV_MAX_RECORD_SIZE', [ + 'Max Record Size:', + 'record exceed the maximum number of tolerated bytes', + `of ${max_record_size}`, + `at line ${this.info.lines}`, + ], this.options, this.__infoField()) + ); + if(err !== undefined) return err; } } - } - if(this.state.commenting === false){ - if(max_record_size !== 0 && this.state.record_length + this.state.field.length > max_record_size){ + const lappend = ltrim === false || this.state.quoting === true || this.state.field.length !== 0 || !this.__isCharTrimable(chr); + // rtrim in non quoting is handle in __onField + const rappend = rtrim === false || this.state.wasQuoting === false; + if(lappend === true && rappend === true){ + this.state.field.append(chr); + }else if(rtrim === true && !this.__isCharTrimable(chr)){ const err = this.__error( - new CsvError('CSV_MAX_RECORD_SIZE', [ - 'Max Record Size:', - 'record exceed the maximum number of tolerated bytes', - `of ${max_record_size}`, + new CsvError('CSV_NON_TRIMABLE_CHAR_AFTER_CLOSING_QUOTE', [ + 'Invalid Closing Quote:', + 'found non trimable byte after quote', `at line ${this.info.lines}`, ], this.options, this.__infoField()) ); if(err !== undefined) return err; } } - const lappend = ltrim === false || this.state.quoting === true || this.state.field.length !== 0 || !this.__isCharTrimable(chr); - // rtrim in non quoting is handle in __onField - const rappend = rtrim === false || this.state.wasQuoting === false; - if(lappend === true && rappend === true){ - this.state.field.append(chr); - }else if(rtrim === true && !this.__isCharTrimable(chr)){ - const err = this.__error( - new CsvError('CSV_NON_TRIMABLE_CHAR_AFTER_CLOSING_QUOTE', [ - 'Invalid Closing Quote:', - 'found non trimable byte after quote', - `at line ${this.info.lines}`, - ], this.options, this.__infoField()) - ); - if(err !== undefined) return err; - } - } - if(end === true){ - // Ensure we are not ending in a quoting state - if(this.state.quoting === true){ - const err = this.__error( - new CsvError('CSV_QUOTE_NOT_CLOSED', [ - 'Quote Not Closed:', - `the parsing is finished with an opening quote at line ${this.info.lines}`, - ], this.options, this.__infoField()) - ); - if(err !== undefined) return err; + if(end === true){ + // Ensure we are not ending in a quoting state + if(this.state.quoting === true){ + const err = this.__error( + new CsvError('CSV_QUOTE_NOT_CLOSED', [ + 'Quote Not Closed:', + `the parsing is finished with an opening quote at line ${this.info.lines}`, + ], this.options, this.__infoField()) + ); + if(err !== undefined) return err; + }else { + // Skip last line if it has no characters + if(this.state.wasQuoting === true || this.state.record.length !== 0 || this.state.field.length !== 0){ + this.info.bytes = this.state.bufBytesStart + pos; + const errField = this.__onField(); + if(errField !== undefined) return errField; + const errRecord = this.__onRecord(push); + if(errRecord !== undefined) return errRecord; + }else if(this.state.wasRowDelimiter === true){ + this.info.empty_lines++; + }else if(this.state.commenting === true){ + this.info.comment_lines++; + } + } }else { - // Skip last line if it has no characters - if(this.state.wasQuoting === true || this.state.record.length !== 0 || this.state.field.length !== 0){ - this.info.bytes = this.state.bufBytesStart + pos; - const errField = this.__onField(); - if(errField !== undefined) return errField; - const errRecord = this.__onRecord(); - if(errRecord !== undefined) return errRecord; - }else if(this.state.wasRowDelimiter === true){ - this.info.empty_lines++; - }else if(this.state.commenting === true){ - this.info.comment_lines++; + this.state.bufBytesStart += pos; + this.state.previousBuf = buf.slice(pos); + } + if(this.state.wasRowDelimiter === true){ + this.info.lines++; + this.state.wasRowDelimiter = false; + } + }, + __onRecord: function(push){ + const {columns, group_columns_by_name, encoding, info, from, relax_column_count, relax_column_count_less, relax_column_count_more, raw, skip_records_with_empty_values} = this.options; + const {enabled, record} = this.state; + if(enabled === false){ + return this.__resetRecord(); + } + // Convert the first line into column names + const recordLength = record.length; + if(columns === true){ + if(skip_records_with_empty_values === true && isRecordEmpty(record)){ + this.__resetRecord(); + return; + } + return this.__firstLineToColumns(record); + } + if(columns === false && this.info.records === 0){ + this.state.expectedRecordLength = recordLength; + } + if(recordLength !== this.state.expectedRecordLength){ + const err = columns === false ? + new CsvError('CSV_RECORD_INCONSISTENT_FIELDS_LENGTH', [ + 'Invalid Record Length:', + `expect ${this.state.expectedRecordLength},`, + `got ${recordLength} on line ${this.info.lines}`, + ], this.options, this.__infoField(), { + record: record, + }) + : + new CsvError('CSV_RECORD_INCONSISTENT_COLUMNS', [ + 'Invalid Record Length:', + `columns length is ${columns.length},`, // rename columns + `got ${recordLength} on line ${this.info.lines}`, + ], this.options, this.__infoField(), { + record: record, + }); + if(relax_column_count === true || + (relax_column_count_less === true && recordLength < this.state.expectedRecordLength) || + (relax_column_count_more === true && recordLength > this.state.expectedRecordLength)){ + this.info.invalid_field_length++; + this.state.error = err; + // Error is undefined with skip_records_with_error + }else { + const finalErr = this.__error(err); + if(finalErr) return finalErr; } } - }else { - this.state.bufBytesStart += pos; - this.state.previousBuf = buf.slice(pos); - } - if(this.state.wasRowDelimiter === true){ - this.info.lines++; - this.state.wasRowDelimiter = false; - } - } - __onRecord(){ - const {columns, group_columns_by_name, encoding, info, from, relax_column_count, relax_column_count_less, relax_column_count_more, raw, skip_records_with_empty_values} = this.options; - const {enabled, record} = this.state; - if(enabled === false){ - return this.__resetRecord(); - } - // Convert the first line into column names - const recordLength = record.length; - if(columns === true){ if(skip_records_with_empty_values === true && isRecordEmpty(record)){ this.__resetRecord(); return; } - return this.__firstLineToColumns(record); - } - if(columns === false && this.info.records === 0){ - this.state.expectedRecordLength = recordLength; - } - if(recordLength !== this.state.expectedRecordLength){ - const err = columns === false ? - new CsvError('CSV_RECORD_INCONSISTENT_FIELDS_LENGTH', [ - 'Invalid Record Length:', - `expect ${this.state.expectedRecordLength},`, - `got ${recordLength} on line ${this.info.lines}`, - ], this.options, this.__infoField(), { - record: record, - }) - : - new CsvError('CSV_RECORD_INCONSISTENT_COLUMNS', [ - 'Invalid Record Length:', - `columns length is ${columns.length},`, // rename columns - `got ${recordLength} on line ${this.info.lines}`, - ], this.options, this.__infoField(), { - record: record, - }); - if(relax_column_count === true || - (relax_column_count_less === true && recordLength < this.state.expectedRecordLength) || - (relax_column_count_more === true && recordLength > this.state.expectedRecordLength)){ - this.info.invalid_field_length++; - this.state.error = err; - // Error is undefined with skip_records_with_error - }else { - const finalErr = this.__error(err); - if(finalErr) return finalErr; + if(this.state.recordHasError === true){ + this.__resetRecord(); + this.state.recordHasError = false; + return; } - } - if(skip_records_with_empty_values === true && isRecordEmpty(record)){ - this.__resetRecord(); - return; - } - if(this.state.recordHasError === true){ - this.__resetRecord(); - this.state.recordHasError = false; - return; - } - this.info.records++; - if(from === 1 || this.info.records >= from){ - const {objname} = this.options; - // With columns, records are object - if(columns !== false){ - const obj = {}; - // Transform record array to an object - for(let i = 0, l = record.length; i < l; i++){ - if(columns[i] === undefined || columns[i].disabled) continue; - // Turn duplicate columns into an array - if (group_columns_by_name === true && obj[columns[i].name] !== undefined) { - if (Array.isArray(obj[columns[i].name])) { - obj[columns[i].name] = obj[columns[i].name].concat(record[i]); + this.info.records++; + if(from === 1 || this.info.records >= from){ + const {objname} = this.options; + // With columns, records are object + if(columns !== false){ + const obj = {}; + // Transform record array to an object + for(let i = 0, l = record.length; i < l; i++){ + if(columns[i] === undefined || columns[i].disabled) continue; + // Turn duplicate columns into an array + if (group_columns_by_name === true && obj[columns[i].name] !== undefined) { + if (Array.isArray(obj[columns[i].name])) { + obj[columns[i].name] = obj[columns[i].name].concat(record[i]); + } else { + obj[columns[i].name] = [obj[columns[i].name], record[i]]; + } } else { - obj[columns[i].name] = [obj[columns[i].name], record[i]]; + obj[columns[i].name] = record[i]; } - } else { - obj[columns[i].name] = record[i]; } - } - // Without objname (default) - if(raw === true || info === true){ - const extRecord = Object.assign( - {record: obj}, - (raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}), - (info === true ? {info: this.__infoRecord()}: {}) - ); - const err = this.__push( - objname === undefined ? extRecord : [obj[objname], extRecord] - ); - if(err){ - return err; - } - }else { - const err = this.__push( - objname === undefined ? obj : [obj[objname], obj] - ); - if(err){ - return err; - } - } - // Without columns, records are array - }else { - if(raw === true || info === true){ - const extRecord = Object.assign( - {record: record}, - raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}, - info === true ? {info: this.__infoRecord()}: {} - ); - const err = this.__push( - objname === undefined ? extRecord : [record[objname], extRecord] - ); - if(err){ - return err; + // Without objname (default) + if(raw === true || info === true){ + const extRecord = Object.assign( + {record: obj}, + (raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}), + (info === true ? {info: this.__infoRecord()}: {}) + ); + const err = this.__push( + objname === undefined ? extRecord : [obj[objname], extRecord] + , push); + if(err){ + return err; + } + }else { + const err = this.__push( + objname === undefined ? obj : [obj[objname], obj] + , push); + if(err){ + return err; + } } + // Without columns, records are array }else { - const err = this.__push( - objname === undefined ? record : [record[objname], record] - ); - if(err){ - return err; + if(raw === true || info === true){ + const extRecord = Object.assign( + {record: record}, + raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}, + info === true ? {info: this.__infoRecord()}: {} + ); + const err = this.__push( + objname === undefined ? extRecord : [record[objname], extRecord] + , push); + if(err){ + return err; + } + }else { + const err = this.__push( + objname === undefined ? record : [record[objname], record] + , push); + if(err){ + return err; + } } } } - } - this.__resetRecord(); - } - __firstLineToColumns(record){ - const {firstLineToHeaders} = this.state; - try{ - const headers = firstLineToHeaders === undefined ? record : firstLineToHeaders.call(null, record); - if(!Array.isArray(headers)){ - return this.__error( - new CsvError('CSV_INVALID_COLUMN_MAPPING', [ - 'Invalid Column Mapping:', - 'expect an array from column function,', - `got ${JSON.stringify(headers)}` - ], this.options, this.__infoField(), { - headers: headers, - }) - ); - } - const normalizedHeaders = normalizeColumnsArray(headers); - this.state.expectedRecordLength = normalizedHeaders.length; - this.options.columns = normalizedHeaders; this.__resetRecord(); - return; - }catch(err){ - return err; - } - } - __resetRecord(){ - if(this.options.raw === true){ - this.state.rawBuffer.reset(); - } - this.state.error = undefined; - this.state.record = []; - this.state.record_length = 0; - } - __onField(){ - const {cast, encoding, rtrim, max_record_size} = this.options; - const {enabled, wasQuoting} = this.state; - // Short circuit for the from_line options - if(enabled === false){ - return this.__resetField(); - } - let field = this.state.field.toString(encoding); - if(rtrim === true && wasQuoting === false){ - field = field.trimRight(); - } - if(cast === true){ - const [err, f] = this.__cast(field); - if(err !== undefined) return err; - field = f; - } - this.state.record.push(field); - // Increment record length if record size must not exceed a limit - if(max_record_size !== 0 && typeof field === 'string'){ - this.state.record_length += field.length; - } - this.__resetField(); - } - __resetField(){ - this.state.field.reset(); - this.state.wasQuoting = false; - } - __push(record){ - const {on_record} = this.options; - if(on_record !== undefined){ - const info = this.__infoRecord(); + }, + __firstLineToColumns: function(record){ + const {firstLineToHeaders} = this.state; try{ - record = on_record.call(null, record, info); + const headers = firstLineToHeaders === undefined ? record : firstLineToHeaders.call(null, record); + if(!Array.isArray(headers)){ + return this.__error( + new CsvError('CSV_INVALID_COLUMN_MAPPING', [ + 'Invalid Column Mapping:', + 'expect an array from column function,', + `got ${JSON.stringify(headers)}` + ], this.options, this.__infoField(), { + headers: headers, + }) + ); + } + const normalizedHeaders = normalize_columns_array(headers); + this.state.expectedRecordLength = normalizedHeaders.length; + this.options.columns = normalizedHeaders; + this.__resetRecord(); + return; }catch(err){ return err; } - if(record === undefined || record === null){ return; } - } - this.push(record); - } - // Return a tuple with the error and the casted value - __cast(field){ - const {columns, relax_column_count} = this.options; - const isColumns = Array.isArray(columns); - // Dont loose time calling cast - // because the final record is an object - // and this field can't be associated to a key present in columns - if(isColumns === true && relax_column_count && this.options.columns.length <= this.state.record.length){ - return [undefined, undefined]; - } - if(this.state.castField !== null){ - try{ + }, + __resetRecord: function(){ + if(this.options.raw === true){ + this.state.rawBuffer.reset(); + } + this.state.error = undefined; + this.state.record = []; + this.state.record_length = 0; + }, + __onField: function(){ + const {cast, encoding, rtrim, max_record_size} = this.options; + const {enabled, wasQuoting} = this.state; + // Short circuit for the from_line options + if(enabled === false){ + return this.__resetField(); + } + let field = this.state.field.toString(encoding); + if(rtrim === true && wasQuoting === false){ + field = field.trimRight(); + } + if(cast === true){ + const [err, f] = this.__cast(field); + if(err !== undefined) return err; + field = f; + } + this.state.record.push(field); + // Increment record length if record size must not exceed a limit + if(max_record_size !== 0 && typeof field === 'string'){ + this.state.record_length += field.length; + } + this.__resetField(); + }, + __resetField: function(){ + this.state.field.reset(); + this.state.wasQuoting = false; + }, + __push: function(record, push){ + const {on_record} = this.options; + if(on_record !== undefined){ + const info = this.__infoRecord(); + try{ + record = on_record.call(null, record, info); + }catch(err){ + return err; + } + if(record === undefined || record === null){ return; } + } + push(record); + }, + // Return a tuple with the error and the casted value + __cast: function(field){ + const {columns, relax_column_count} = this.options; + const isColumns = Array.isArray(columns); + // Dont loose time calling cast + // because the final record is an object + // and this field can't be associated to a key present in columns + if(isColumns === true && relax_column_count && this.options.columns.length <= this.state.record.length){ + return [undefined, undefined]; + } + if(this.state.castField !== null){ + try{ + const info = this.__infoField(); + return [undefined, this.state.castField.call(null, field, info)]; + }catch(err){ + return [err]; + } + } + if(this.__isFloat(field)){ + return [undefined, parseFloat(field)]; + }else if(this.options.cast_date !== false){ const info = this.__infoField(); - return [undefined, this.state.castField.call(null, field, info)]; - }catch(err){ - return [err]; + return [undefined, this.options.cast_date.call(null, field, info)]; } - } - if(this.__isFloat(field)){ - return [undefined, parseFloat(field)]; - }else if(this.options.cast_date !== false){ - const info = this.__infoField(); - return [undefined, this.options.cast_date.call(null, field, info)]; - } - return [undefined, field]; - } - // Helper to test if a character is a space or a line delimiter - __isCharTrimable(chr){ - return chr === space || chr === tab || chr === cr || chr === nl || chr === np; - } - // Keep it in case we implement the `cast_int` option - // __isInt(value){ - // // return Number.isInteger(parseInt(value)) - // // return !isNaN( parseInt( obj ) ); - // return /^(\-|\+)?[1-9][0-9]*$/.test(value) - // } - __isFloat(value){ - return (value - parseFloat(value) + 1) >= 0; // Borrowed from jquery - } - __compareBytes(sourceBuf, targetBuf, targetPos, firstByte){ - if(sourceBuf[0] !== firstByte) return 0; - const sourceLength = sourceBuf.length; - for(let i = 1; i < sourceLength; i++){ - if(sourceBuf[i] !== targetBuf[targetPos+i]) return 0; - } - return sourceLength; - } - __needMoreData(i, bufLen, end){ - if(end) return false; - const {quote} = this.options; - const {quoting, needMoreDataSize, recordDelimiterMaxLength} = this.state; - const numOfCharLeft = bufLen - i - 1; - const requiredLength = Math.max( - needMoreDataSize, - // Skip if the remaining buffer smaller than record delimiter - recordDelimiterMaxLength, - // Skip if the remaining buffer can be record delimiter following the closing quote - // 1 is for quote.length - quoting ? (quote.length + recordDelimiterMaxLength) : 0, - ); - return numOfCharLeft < requiredLength; - } - __isDelimiter(buf, pos, chr){ - const {delimiter, ignore_last_delimiters} = this.options; - if(ignore_last_delimiters === true && this.state.record.length === this.options.columns.length - 1){ - return 0; - }else if(ignore_last_delimiters !== false && typeof ignore_last_delimiters === 'number' && this.state.record.length === ignore_last_delimiters - 1){ - return 0; - } - loop1: for(let i = 0; i < delimiter.length; i++){ - const del = delimiter[i]; - if(del[0] === chr){ - for(let j = 1; j < del.length; j++){ - if(del[j] !== buf[pos+j]) continue loop1; + return [undefined, field]; + }, + // Helper to test if a character is a space or a line delimiter + __isCharTrimable: function(chr){ + return chr === space || chr === tab || chr === cr || chr === nl || chr === np; + }, + // Keep it in case we implement the `cast_int` option + // __isInt(value){ + // // return Number.isInteger(parseInt(value)) + // // return !isNaN( parseInt( obj ) ); + // return /^(\-|\+)?[1-9][0-9]*$/.test(value) + // } + __isFloat: function(value){ + return (value - parseFloat(value) + 1) >= 0; // Borrowed from jquery + }, + __compareBytes: function(sourceBuf, targetBuf, targetPos, firstByte){ + if(sourceBuf[0] !== firstByte) return 0; + const sourceLength = sourceBuf.length; + for(let i = 1; i < sourceLength; i++){ + if(sourceBuf[i] !== targetBuf[targetPos+i]) return 0; + } + return sourceLength; + }, + __isDelimiter: function(buf, pos, chr){ + const {delimiter, ignore_last_delimiters} = this.options; + if(ignore_last_delimiters === true && this.state.record.length === this.options.columns.length - 1){ + return 0; + }else if(ignore_last_delimiters !== false && typeof ignore_last_delimiters === 'number' && this.state.record.length === ignore_last_delimiters - 1){ + return 0; + } + loop1: for(let i = 0; i < delimiter.length; i++){ + const del = delimiter[i]; + if(del[0] === chr){ + for(let j = 1; j < del.length; j++){ + if(del[j] !== buf[pos+j]) continue loop1; + } + return del.length; } - return del.length; } - } - return 0; - } - __isRecordDelimiter(chr, buf, pos){ - const {record_delimiter} = this.options; - const recordDelimiterLength = record_delimiter.length; - loop1: for(let i = 0; i < recordDelimiterLength; i++){ - const rd = record_delimiter[i]; - const rdLength = rd.length; - if(rd[0] !== chr){ - continue; + return 0; + }, + __isRecordDelimiter: function(chr, buf, pos){ + const {record_delimiter} = this.options; + const recordDelimiterLength = record_delimiter.length; + loop1: for(let i = 0; i < recordDelimiterLength; i++){ + const rd = record_delimiter[i]; + const rdLength = rd.length; + if(rd[0] !== chr){ + continue; + } + for(let j = 1; j < rdLength; j++){ + if(rd[j] !== buf[pos+j]){ + continue loop1; + } + } + return rd.length; } - for(let j = 1; j < rdLength; j++){ - if(rd[j] !== buf[pos+j]){ - continue loop1; + return 0; + }, + __isEscape: function(buf, pos, chr){ + const {escape} = this.options; + if(escape === null) return false; + const l = escape.length; + if(escape[0] === chr){ + for(let i = 0; i < l; i++){ + if(escape[i] !== buf[pos+i]){ + return false; + } } + return true; } - return rd.length; - } - return 0; - } - __isEscape(buf, pos, chr){ - const {escape} = this.options; - if(escape === null) return false; - const l = escape.length; - if(escape[0] === chr){ + return false; + }, + __isQuote: function(buf, pos){ + const {quote} = this.options; + if(quote === null) return false; + const l = quote.length; for(let i = 0; i < l; i++){ - if(escape[i] !== buf[pos+i]){ + if(quote[i] !== buf[pos+i]){ return false; } } return true; - } - return false; - } - __isQuote(buf, pos){ - const {quote} = this.options; - if(quote === null) return false; - const l = quote.length; - for(let i = 0; i < l; i++){ - if(quote[i] !== buf[pos+i]){ - return false; - } - } - return true; - } - __autoDiscoverRecordDelimiter(buf, pos){ - const {encoding} = this.options; - const chr = buf[pos]; - if(chr === cr){ - if(buf[pos+1] === nl){ - this.options.record_delimiter.push(Buffer.from('\r\n', encoding)); - this.state.recordDelimiterMaxLength = 2; - return 2; - }else { - this.options.record_delimiter.push(Buffer.from('\r', encoding)); + }, + __autoDiscoverRecordDelimiter: function(buf, pos){ + const {encoding} = this.options; + const chr = buf[pos]; + if(chr === cr){ + if(buf[pos+1] === nl){ + this.options.record_delimiter.push(Buffer.from('\r\n', encoding)); + this.state.recordDelimiterMaxLength = 2; + return 2; + }else { + this.options.record_delimiter.push(Buffer.from('\r', encoding)); + this.state.recordDelimiterMaxLength = 1; + return 1; + } + }else if(chr === nl){ + this.options.record_delimiter.push(Buffer.from('\n', encoding)); this.state.recordDelimiterMaxLength = 1; return 1; } - }else if(chr === nl){ - this.options.record_delimiter.push(Buffer.from('\n', encoding)); - this.state.recordDelimiterMaxLength = 1; - return 1; - } - return 0; - } - __error(msg){ - const {encoding, raw, skip_records_with_error} = this.options; - const err = typeof msg === 'string' ? new Error(msg) : msg; - if(skip_records_with_error){ - this.state.recordHasError = true; - this.emit('skip', err, raw ? this.state.rawBuffer.toString(encoding) : undefined); - return undefined; - }else { - return err; + return 0; + }, + __error: function(msg){ + const {encoding, raw, skip_records_with_error} = this.options; + const err = typeof msg === 'string' ? new Error(msg) : msg; + if(skip_records_with_error){ + this.state.recordHasError = true; + if(this.options.on_skip !== undefined){ + this.options.on_skip(err, raw ? this.state.rawBuffer.toString(encoding) : undefined); + } + // this.emit('skip', err, raw ? this.state.rawBuffer.toString(encoding) : undefined); + return undefined; + }else { + return err; + } + }, + __infoDataSet: function(){ + return { + ...this.info, + columns: this.options.columns + }; + }, + __infoRecord: function(){ + const {columns, raw, encoding} = this.options; + return { + ...this.__infoDataSet(), + error: this.state.error, + header: columns === true, + index: this.state.record.length, + raw: raw ? this.state.rawBuffer.toString(encoding) : undefined + }; + }, + __infoField: function(){ + const {columns} = this.options; + const isColumns = Array.isArray(columns); + return { + ...this.__infoRecord(), + column: isColumns === true ? + (columns.length > this.state.record.length ? + columns[this.state.record.length].name : + null + ) : + this.state.record.length, + quoting: this.state.wasQuoting, + }; } - } - __infoDataSet(){ - return { - ...this.info, - columns: this.options.columns + }; +}; + +class Parser extends stream.Transform { + constructor(opts = {}){ + super({...{readableObjectMode: true}, ...opts, encoding: null}); + this.api = transform(opts); + this.api.options.on_skip = (err, chunk) => { + this.emit('skip', err, chunk); }; + // Backward compatibility + this.state = this.api.state; + this.options = this.api.options; + this.info = this.api.info; } - __infoRecord(){ - const {columns, raw, encoding} = this.options; - return { - ...this.__infoDataSet(), - error: this.state.error, - header: columns === true, - index: this.state.record.length, - raw: raw ? this.state.rawBuffer.toString(encoding) : undefined - }; + // Implementation of `Transform._transform` + _transform(buf, encoding, callback){ + if(this.state.stop === true){ + return; + } + const err = this.api.parse(buf, false, (record) => { + this.push.call(this, record); + }, () => { + this.push.call(this, null); + }); + if(err !== undefined){ + this.state.stop = true; + } + callback(err); } - __infoField(){ - const {columns} = this.options; - const isColumns = Array.isArray(columns); - return { - ...this.__infoRecord(), - column: isColumns === true ? - (columns.length > this.state.record.length ? - columns[this.state.record.length].name : - null - ) : - this.state.record.length, - quoting: this.state.wasQuoting, - }; + // Implementation of `Transform._flush` + _flush(callback){ + if(this.state.stop === true){ + return; + } + const err = this.api.parse(undefined, true, (record) => { + this.push.call(this, record); + }, () => { + this.push.call(this, null); + }); + callback(err); } } @@ -1280,7 +1311,7 @@ const parse = function(){ const type = typeof argument; if(data === undefined && (typeof argument === 'string' || Buffer.isBuffer(argument))){ data = argument; - }else if(options === undefined && isObject(argument)){ + }else if(options === undefined && is_object(argument)){ options = argument; }else if(callback === undefined && type === 'function'){ callback = argument; @@ -1305,10 +1336,10 @@ const parse = function(){ } }); parser.on('error', function(err){ - callback(err, undefined, parser.__infoDataSet()); + callback(err, undefined, parser.api.__infoDataSet()); }); parser.on('end', function(){ - callback(undefined, records, parser.__infoDataSet()); + callback(undefined, records, parser.api.__infoDataSet()); }); } if(data !== undefined){ diff --git a/packages/csv-parse/dist/cjs/sync.cjs b/packages/csv-parse/dist/cjs/sync.cjs index 29e5875f7..958432156 100644 --- a/packages/csv-parse/dist/cjs/sync.cjs +++ b/packages/csv-parse/dist/cjs/sync.cjs @@ -2,7 +2,54 @@ Object.defineProperty(exports, '__esModule', { value: true }); -var stream = require('stream'); +class CsvError extends Error { + constructor(code, message, options, ...contexts) { + if(Array.isArray(message)) message = message.join(' '); + super(message); + if(Error.captureStackTrace !== undefined){ + Error.captureStackTrace(this, CsvError); + } + this.code = code; + for(const context of contexts){ + for(const key in context){ + const value = context[key]; + this[key] = Buffer.isBuffer(value) ? value.toString(options.encoding) : value == null ? value : JSON.parse(JSON.stringify(value)); + } + } + } +} + +const is_object = function(obj){ + return (typeof obj === 'object' && obj !== null && !Array.isArray(obj)); +}; + +const normalize_columns_array = function(columns){ + const normalizedColumns = []; + for(let i = 0, l = columns.length; i < l; i++){ + const column = columns[i]; + if(column === undefined || column === null || column === false){ + normalizedColumns[i] = { disabled: true }; + }else if(typeof column === 'string'){ + normalizedColumns[i] = { name: column }; + }else if(is_object(column)){ + if(typeof column.name !== 'string'){ + throw new CsvError('CSV_OPTION_COLUMNS_MISSING_NAME', [ + 'Option columns missing name:', + `property "name" is required at position ${i}`, + 'when column is an object literal' + ]); + } + normalizedColumns[i] = column; + }else { + throw new CsvError('CSV_INVALID_COLUMN_DEFINITION', [ + 'Invalid column definition:', + 'expect a string or a literal object,', + `got ${JSON.stringify(column)} at position ${i}` + ]); + } + } + return normalizedColumns; +}; class ResizeableBuffer{ constructor(size=100){ @@ -66,1232 +113,1171 @@ class ResizeableBuffer{ } } -// white space characters -// https://en.wikipedia.org/wiki/Whitespace_character -// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions/Character_Classes#Types -// \f\n\r\t\v\u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff -const tab = 9; -const nl = 10; // \n, 0x0A in hexadecimal, 10 in decimal -const np = 12; -const cr = 13; // \r, 0x0D in hexadécimal, 13 in decimal -const space = 32; -const boms = { - // Note, the following are equals: - // Buffer.from("\ufeff") - // Buffer.from([239, 187, 191]) - // Buffer.from('EFBBBF', 'hex') - 'utf8': Buffer.from([239, 187, 191]), - // Note, the following are equals: - // Buffer.from "\ufeff", 'utf16le - // Buffer.from([255, 254]) - 'utf16le': Buffer.from([255, 254]) +const init_state = function(options){ + return { + bomSkipped: false, + bufBytesStart: 0, + castField: options.cast_function, + commenting: false, + // Current error encountered by a record + error: undefined, + enabled: options.from_line === 1, + escaping: false, + escapeIsQuote: Buffer.isBuffer(options.escape) && Buffer.isBuffer(options.quote) && Buffer.compare(options.escape, options.quote) === 0, + // columns can be `false`, `true`, `Array` + expectedRecordLength: Array.isArray(options.columns) ? options.columns.length : undefined, + field: new ResizeableBuffer(20), + firstLineToHeaders: options.cast_first_line_to_header, + needMoreDataSize: Math.max( + // Skip if the remaining buffer smaller than comment + options.comment !== null ? options.comment.length : 0, + // Skip if the remaining buffer can be delimiter + ...options.delimiter.map((delimiter) => delimiter.length), + // Skip if the remaining buffer can be escape sequence + options.quote !== null ? options.quote.length : 0, + ), + previousBuf: undefined, + quoting: false, + stop: false, + rawBuffer: new ResizeableBuffer(100), + record: [], + recordHasError: false, + record_length: 0, + recordDelimiterMaxLength: options.record_delimiter.length === 0 ? 2 : Math.max(...options.record_delimiter.map((v) => v.length)), + trimChars: [Buffer.from(' ', options.encoding)[0], Buffer.from('\t', options.encoding)[0]], + wasQuoting: false, + wasRowDelimiter: false + }; }; -class CsvError extends Error { - constructor(code, message, options, ...contexts) { - if(Array.isArray(message)) message = message.join(' '); - super(message); - if(Error.captureStackTrace !== undefined){ - Error.captureStackTrace(this, CsvError); - } - this.code = code; - for(const context of contexts){ - for(const key in context){ - const value = context[key]; - this[key] = Buffer.isBuffer(value) ? value.toString(options.encoding) : value == null ? value : JSON.parse(JSON.stringify(value)); - } - } - } -} - const underscore = function(str){ return str.replace(/([A-Z])/g, function(_, match){ return '_' + match.toLowerCase(); }); }; -const isObject = function(obj){ - return (typeof obj === 'object' && obj !== null && !Array.isArray(obj)); -}; - -const isRecordEmpty = function(record){ - return record.every((field) => field == null || field.toString && field.toString().trim() === ''); -}; - -const normalizeColumnsArray = function(columns){ - const normalizedColumns = []; - for(let i = 0, l = columns.length; i < l; i++){ - const column = columns[i]; - if(column === undefined || column === null || column === false){ - normalizedColumns[i] = { disabled: true }; - }else if(typeof column === 'string'){ - normalizedColumns[i] = { name: column }; - }else if(isObject(column)){ - if(typeof column.name !== 'string'){ - throw new CsvError('CSV_OPTION_COLUMNS_MISSING_NAME', [ - 'Option columns missing name:', - `property "name" is required at position ${i}`, - 'when column is an object literal' - ]); - } - normalizedColumns[i] = column; - }else { - throw new CsvError('CSV_INVALID_COLUMN_DEFINITION', [ - 'Invalid column definition:', - 'expect a string or a literal object,', - `got ${JSON.stringify(column)} at position ${i}` - ]); - } +const normalize_options = function(opts){ + const options = {}; + // Merge with user options + for(const opt in opts){ + options[underscore(opt)] = opts[opt]; } - return normalizedColumns; -}; - -class Parser extends stream.Transform { - constructor(opts = {}){ - super({...{readableObjectMode: true}, ...opts, encoding: null}); - this.__originalOptions = opts; - this.__normalizeOptions(opts); + // Normalize option `encoding` + // Note: defined first because other options depends on it + // to convert chars/strings into buffers. + if(options.encoding === undefined || options.encoding === true){ + options.encoding = 'utf8'; + }else if(options.encoding === null || options.encoding === false){ + options.encoding = null; + }else if(typeof options.encoding !== 'string' && options.encoding !== null){ + throw new CsvError('CSV_INVALID_OPTION_ENCODING', [ + 'Invalid option encoding:', + 'encoding must be a string or null to return a buffer,', + `got ${JSON.stringify(options.encoding)}` + ], options); } - __normalizeOptions(opts){ - const options = {}; - // Merge with user options - for(const opt in opts){ - options[underscore(opt)] = opts[opt]; - } - // Normalize option `encoding` - // Note: defined first because other options depends on it - // to convert chars/strings into buffers. - if(options.encoding === undefined || options.encoding === true){ - options.encoding = 'utf8'; - }else if(options.encoding === null || options.encoding === false){ - options.encoding = null; - }else if(typeof options.encoding !== 'string' && options.encoding !== null){ - throw new CsvError('CSV_INVALID_OPTION_ENCODING', [ - 'Invalid option encoding:', - 'encoding must be a string or null to return a buffer,', - `got ${JSON.stringify(options.encoding)}` - ], options); - } - // Normalize option `bom` - if(options.bom === undefined || options.bom === null || options.bom === false){ - options.bom = false; - }else if(options.bom !== true){ - throw new CsvError('CSV_INVALID_OPTION_BOM', [ - 'Invalid option bom:', 'bom must be true,', - `got ${JSON.stringify(options.bom)}` - ], options); - } - // Normalize option `cast` - let fnCastField = null; - if(options.cast === undefined || options.cast === null || options.cast === false || options.cast === ''){ - options.cast = undefined; - }else if(typeof options.cast === 'function'){ - fnCastField = options.cast; - options.cast = true; - }else if(options.cast !== true){ - throw new CsvError('CSV_INVALID_OPTION_CAST', [ - 'Invalid option cast:', 'cast must be true or a function,', - `got ${JSON.stringify(options.cast)}` - ], options); - } - // Normalize option `cast_date` - if(options.cast_date === undefined || options.cast_date === null || options.cast_date === false || options.cast_date === ''){ - options.cast_date = false; - }else if(options.cast_date === true){ - options.cast_date = function(value){ - const date = Date.parse(value); - return !isNaN(date) ? new Date(date) : value; - }; - }else { - throw new CsvError('CSV_INVALID_OPTION_CAST_DATE', [ - 'Invalid option cast_date:', 'cast_date must be true or a function,', - `got ${JSON.stringify(options.cast_date)}` + // Normalize option `bom` + if(options.bom === undefined || options.bom === null || options.bom === false){ + options.bom = false; + }else if(options.bom !== true){ + throw new CsvError('CSV_INVALID_OPTION_BOM', [ + 'Invalid option bom:', 'bom must be true,', + `got ${JSON.stringify(options.bom)}` + ], options); + } + // Normalize option `cast` + options.cast_function = null; + if(options.cast === undefined || options.cast === null || options.cast === false || options.cast === ''){ + options.cast = undefined; + }else if(typeof options.cast === 'function'){ + options.cast_function = options.cast; + options.cast = true; + }else if(options.cast !== true){ + throw new CsvError('CSV_INVALID_OPTION_CAST', [ + 'Invalid option cast:', 'cast must be true or a function,', + `got ${JSON.stringify(options.cast)}` + ], options); + } + // Normalize option `cast_date` + if(options.cast_date === undefined || options.cast_date === null || options.cast_date === false || options.cast_date === ''){ + options.cast_date = false; + }else if(options.cast_date === true){ + options.cast_date = function(value){ + const date = Date.parse(value); + return !isNaN(date) ? new Date(date) : value; + }; + }else { + throw new CsvError('CSV_INVALID_OPTION_CAST_DATE', [ + 'Invalid option cast_date:', 'cast_date must be true or a function,', + `got ${JSON.stringify(options.cast_date)}` + ], options); + } + // Normalize option `columns` + options.cast_first_line_to_header = null; + if(options.columns === true){ + // Fields in the first line are converted as-is to columns + options.cast_first_line_to_header = undefined; + }else if(typeof options.columns === 'function'){ + options.cast_first_line_to_header = options.columns; + options.columns = true; + }else if(Array.isArray(options.columns)){ + options.columns = normalize_columns_array(options.columns); + }else if(options.columns === undefined || options.columns === null || options.columns === false){ + options.columns = false; + }else { + throw new CsvError('CSV_INVALID_OPTION_COLUMNS', [ + 'Invalid option columns:', + 'expect an array, a function or true,', + `got ${JSON.stringify(options.columns)}` + ], options); + } + // Normalize option `group_columns_by_name` + if(options.group_columns_by_name === undefined || options.group_columns_by_name === null || options.group_columns_by_name === false){ + options.group_columns_by_name = false; + }else if(options.group_columns_by_name !== true){ + throw new CsvError('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ + 'Invalid option group_columns_by_name:', + 'expect an boolean,', + `got ${JSON.stringify(options.group_columns_by_name)}` + ], options); + }else if(options.columns === false){ + throw new CsvError('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ + 'Invalid option group_columns_by_name:', + 'the `columns` mode must be activated.' + ], options); + } + // Normalize option `comment` + if(options.comment === undefined || options.comment === null || options.comment === false || options.comment === ''){ + options.comment = null; + }else { + if(typeof options.comment === 'string'){ + options.comment = Buffer.from(options.comment, options.encoding); + } + if(!Buffer.isBuffer(options.comment)){ + throw new CsvError('CSV_INVALID_OPTION_COMMENT', [ + 'Invalid option comment:', + 'comment must be a buffer or a string,', + `got ${JSON.stringify(options.comment)}` ], options); } - // Normalize option `columns` - let fnFirstLineToHeaders = null; - if(options.columns === true){ - // Fields in the first line are converted as-is to columns - fnFirstLineToHeaders = undefined; - }else if(typeof options.columns === 'function'){ - fnFirstLineToHeaders = options.columns; - options.columns = true; - }else if(Array.isArray(options.columns)){ - options.columns = normalizeColumnsArray(options.columns); - }else if(options.columns === undefined || options.columns === null || options.columns === false){ - options.columns = false; - }else { - throw new CsvError('CSV_INVALID_OPTION_COLUMNS', [ - 'Invalid option columns:', - 'expect an array, a function or true,', - `got ${JSON.stringify(options.columns)}` - ], options); + } + // Normalize option `delimiter` + const delimiter_json = JSON.stringify(options.delimiter); + if(!Array.isArray(options.delimiter)) options.delimiter = [options.delimiter]; + if(options.delimiter.length === 0){ + throw new CsvError('CSV_INVALID_OPTION_DELIMITER', [ + 'Invalid option delimiter:', + 'delimiter must be a non empty string or buffer or array of string|buffer,', + `got ${delimiter_json}` + ], options); + } + options.delimiter = options.delimiter.map(function(delimiter){ + if(delimiter === undefined || delimiter === null || delimiter === false){ + return Buffer.from(',', options.encoding); } - // Normalize option `group_columns_by_name` - if(options.group_columns_by_name === undefined || options.group_columns_by_name === null || options.group_columns_by_name === false){ - options.group_columns_by_name = false; - }else if(options.group_columns_by_name !== true){ - throw new CsvError('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ - 'Invalid option group_columns_by_name:', - 'expect an boolean,', - `got ${JSON.stringify(options.group_columns_by_name)}` - ], options); - }else if(options.columns === false){ - throw new CsvError('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ - 'Invalid option group_columns_by_name:', - 'the `columns` mode must be activated.' - ], options); + if(typeof delimiter === 'string'){ + delimiter = Buffer.from(delimiter, options.encoding); } - // Normalize option `comment` - if(options.comment === undefined || options.comment === null || options.comment === false || options.comment === ''){ - options.comment = null; - }else { - if(typeof options.comment === 'string'){ - options.comment = Buffer.from(options.comment, options.encoding); - } - if(!Buffer.isBuffer(options.comment)){ - throw new CsvError('CSV_INVALID_OPTION_COMMENT', [ - 'Invalid option comment:', - 'comment must be a buffer or a string,', - `got ${JSON.stringify(options.comment)}` - ], options); - } - } - // Normalize option `delimiter` - const delimiter_json = JSON.stringify(options.delimiter); - if(!Array.isArray(options.delimiter)) options.delimiter = [options.delimiter]; - if(options.delimiter.length === 0){ + if(!Buffer.isBuffer(delimiter) || delimiter.length === 0){ throw new CsvError('CSV_INVALID_OPTION_DELIMITER', [ 'Invalid option delimiter:', 'delimiter must be a non empty string or buffer or array of string|buffer,', `got ${delimiter_json}` ], options); } - options.delimiter = options.delimiter.map(function(delimiter){ - if(delimiter === undefined || delimiter === null || delimiter === false){ - return Buffer.from(',', options.encoding); - } - if(typeof delimiter === 'string'){ - delimiter = Buffer.from(delimiter, options.encoding); - } - if(!Buffer.isBuffer(delimiter) || delimiter.length === 0){ - throw new CsvError('CSV_INVALID_OPTION_DELIMITER', [ - 'Invalid option delimiter:', - 'delimiter must be a non empty string or buffer or array of string|buffer,', - `got ${delimiter_json}` - ], options); - } - return delimiter; - }); - // Normalize option `escape` - if(options.escape === undefined || options.escape === true){ - options.escape = Buffer.from('"', options.encoding); - }else if(typeof options.escape === 'string'){ - options.escape = Buffer.from(options.escape, options.encoding); - }else if (options.escape === null || options.escape === false){ - options.escape = null; + return delimiter; + }); + // Normalize option `escape` + if(options.escape === undefined || options.escape === true){ + options.escape = Buffer.from('"', options.encoding); + }else if(typeof options.escape === 'string'){ + options.escape = Buffer.from(options.escape, options.encoding); + }else if (options.escape === null || options.escape === false){ + options.escape = null; + } + if(options.escape !== null){ + if(!Buffer.isBuffer(options.escape)){ + throw new Error(`Invalid Option: escape must be a buffer, a string or a boolean, got ${JSON.stringify(options.escape)}`); } - if(options.escape !== null){ - if(!Buffer.isBuffer(options.escape)){ - throw new Error(`Invalid Option: escape must be a buffer, a string or a boolean, got ${JSON.stringify(options.escape)}`); + } + // Normalize option `from` + if(options.from === undefined || options.from === null){ + options.from = 1; + }else { + if(typeof options.from === 'string' && /\d+/.test(options.from)){ + options.from = parseInt(options.from); + } + if(Number.isInteger(options.from)){ + if(options.from < 0){ + throw new Error(`Invalid Option: from must be a positive integer, got ${JSON.stringify(opts.from)}`); } - } - // Normalize option `from` - if(options.from === undefined || options.from === null){ - options.from = 1; }else { - if(typeof options.from === 'string' && /\d+/.test(options.from)){ - options.from = parseInt(options.from); - } - if(Number.isInteger(options.from)){ - if(options.from < 0){ - throw new Error(`Invalid Option: from must be a positive integer, got ${JSON.stringify(opts.from)}`); - } - }else { - throw new Error(`Invalid Option: from must be an integer, got ${JSON.stringify(options.from)}`); - } + throw new Error(`Invalid Option: from must be an integer, got ${JSON.stringify(options.from)}`); } - // Normalize option `from_line` - if(options.from_line === undefined || options.from_line === null){ - options.from_line = 1; - }else { - if(typeof options.from_line === 'string' && /\d+/.test(options.from_line)){ - options.from_line = parseInt(options.from_line); - } - if(Number.isInteger(options.from_line)){ - if(options.from_line <= 0){ - throw new Error(`Invalid Option: from_line must be a positive integer greater than 0, got ${JSON.stringify(opts.from_line)}`); - } - }else { - throw new Error(`Invalid Option: from_line must be an integer, got ${JSON.stringify(opts.from_line)}`); - } - } - // Normalize options `ignore_last_delimiters` - if(options.ignore_last_delimiters === undefined || options.ignore_last_delimiters === null){ - options.ignore_last_delimiters = false; - }else if(typeof options.ignore_last_delimiters === 'number'){ - options.ignore_last_delimiters = Math.floor(options.ignore_last_delimiters); - if(options.ignore_last_delimiters === 0){ - options.ignore_last_delimiters = false; + } + // Normalize option `from_line` + if(options.from_line === undefined || options.from_line === null){ + options.from_line = 1; + }else { + if(typeof options.from_line === 'string' && /\d+/.test(options.from_line)){ + options.from_line = parseInt(options.from_line); + } + if(Number.isInteger(options.from_line)){ + if(options.from_line <= 0){ + throw new Error(`Invalid Option: from_line must be a positive integer greater than 0, got ${JSON.stringify(opts.from_line)}`); } - }else if(typeof options.ignore_last_delimiters !== 'boolean'){ - throw new CsvError('CSV_INVALID_OPTION_IGNORE_LAST_DELIMITERS', [ - 'Invalid option `ignore_last_delimiters`:', - 'the value must be a boolean value or an integer,', - `got ${JSON.stringify(options.ignore_last_delimiters)}` - ], options); - } - if(options.ignore_last_delimiters === true && options.columns === false){ - throw new CsvError('CSV_IGNORE_LAST_DELIMITERS_REQUIRES_COLUMNS', [ - 'The option `ignore_last_delimiters`', - 'requires the activation of the `columns` option' - ], options); - } - // Normalize option `info` - if(options.info === undefined || options.info === null || options.info === false){ - options.info = false; - }else if(options.info !== true){ - throw new Error(`Invalid Option: info must be true, got ${JSON.stringify(options.info)}`); - } - // Normalize option `max_record_size` - if(options.max_record_size === undefined || options.max_record_size === null || options.max_record_size === false){ - options.max_record_size = 0; - }else if(Number.isInteger(options.max_record_size) && options.max_record_size >= 0);else if(typeof options.max_record_size === 'string' && /\d+/.test(options.max_record_size)){ - options.max_record_size = parseInt(options.max_record_size); }else { - throw new Error(`Invalid Option: max_record_size must be a positive integer, got ${JSON.stringify(options.max_record_size)}`); - } - // Normalize option `objname` - if(options.objname === undefined || options.objname === null || options.objname === false){ - options.objname = undefined; - }else if(Buffer.isBuffer(options.objname)){ - if(options.objname.length === 0){ - throw new Error(`Invalid Option: objname must be a non empty buffer`); - } - if(options.encoding === null);else { - options.objname = options.objname.toString(options.encoding); - } - }else if(typeof options.objname === 'string'){ - if(options.objname.length === 0){ - throw new Error(`Invalid Option: objname must be a non empty string`); - } - // Great, nothing to do - }else if(typeof options.objname === 'number');else { - throw new Error(`Invalid Option: objname must be a string or a buffer, got ${options.objname}`); + throw new Error(`Invalid Option: from_line must be an integer, got ${JSON.stringify(opts.from_line)}`); } - if(options.objname !== undefined){ - if(typeof options.objname === 'number'){ - if(options.columns !== false){ - throw Error('Invalid Option: objname index cannot be combined with columns or be defined as a field'); - } - }else { // A string or a buffer - if(options.columns === false){ - throw Error('Invalid Option: objname field must be combined with columns or be defined as an index'); - } - } - } - // Normalize option `on_record` - if(options.on_record === undefined || options.on_record === null){ - options.on_record = undefined; - }else if(typeof options.on_record !== 'function'){ - throw new CsvError('CSV_INVALID_OPTION_ON_RECORD', [ - 'Invalid option `on_record`:', - 'expect a function,', - `got ${JSON.stringify(options.on_record)}` - ], options); + } + // Normalize options `ignore_last_delimiters` + if(options.ignore_last_delimiters === undefined || options.ignore_last_delimiters === null){ + options.ignore_last_delimiters = false; + }else if(typeof options.ignore_last_delimiters === 'number'){ + options.ignore_last_delimiters = Math.floor(options.ignore_last_delimiters); + if(options.ignore_last_delimiters === 0){ + options.ignore_last_delimiters = false; } - // Normalize option `quote` - if(options.quote === null || options.quote === false || options.quote === ''){ - options.quote = null; - }else { - if(options.quote === undefined || options.quote === true){ - options.quote = Buffer.from('"', options.encoding); - }else if(typeof options.quote === 'string'){ - options.quote = Buffer.from(options.quote, options.encoding); + }else if(typeof options.ignore_last_delimiters !== 'boolean'){ + throw new CsvError('CSV_INVALID_OPTION_IGNORE_LAST_DELIMITERS', [ + 'Invalid option `ignore_last_delimiters`:', + 'the value must be a boolean value or an integer,', + `got ${JSON.stringify(options.ignore_last_delimiters)}` + ], options); + } + if(options.ignore_last_delimiters === true && options.columns === false){ + throw new CsvError('CSV_IGNORE_LAST_DELIMITERS_REQUIRES_COLUMNS', [ + 'The option `ignore_last_delimiters`', + 'requires the activation of the `columns` option' + ], options); + } + // Normalize option `info` + if(options.info === undefined || options.info === null || options.info === false){ + options.info = false; + }else if(options.info !== true){ + throw new Error(`Invalid Option: info must be true, got ${JSON.stringify(options.info)}`); + } + // Normalize option `max_record_size` + if(options.max_record_size === undefined || options.max_record_size === null || options.max_record_size === false){ + options.max_record_size = 0; + }else if(Number.isInteger(options.max_record_size) && options.max_record_size >= 0);else if(typeof options.max_record_size === 'string' && /\d+/.test(options.max_record_size)){ + options.max_record_size = parseInt(options.max_record_size); + }else { + throw new Error(`Invalid Option: max_record_size must be a positive integer, got ${JSON.stringify(options.max_record_size)}`); + } + // Normalize option `objname` + if(options.objname === undefined || options.objname === null || options.objname === false){ + options.objname = undefined; + }else if(Buffer.isBuffer(options.objname)){ + if(options.objname.length === 0){ + throw new Error(`Invalid Option: objname must be a non empty buffer`); + } + if(options.encoding === null);else { + options.objname = options.objname.toString(options.encoding); + } + }else if(typeof options.objname === 'string'){ + if(options.objname.length === 0){ + throw new Error(`Invalid Option: objname must be a non empty string`); + } + // Great, nothing to do + }else if(typeof options.objname === 'number');else { + throw new Error(`Invalid Option: objname must be a string or a buffer, got ${options.objname}`); + } + if(options.objname !== undefined){ + if(typeof options.objname === 'number'){ + if(options.columns !== false){ + throw Error('Invalid Option: objname index cannot be combined with columns or be defined as a field'); } - if(!Buffer.isBuffer(options.quote)){ - throw new Error(`Invalid Option: quote must be a buffer or a string, got ${JSON.stringify(options.quote)}`); + }else { // A string or a buffer + if(options.columns === false){ + throw Error('Invalid Option: objname field must be combined with columns or be defined as an index'); } } - // Normalize option `raw` - if(options.raw === undefined || options.raw === null || options.raw === false){ - options.raw = false; - }else if(options.raw !== true){ - throw new Error(`Invalid Option: raw must be true, got ${JSON.stringify(options.raw)}`); + } + // Normalize option `on_record` + if(options.on_record === undefined || options.on_record === null){ + options.on_record = undefined; + }else if(typeof options.on_record !== 'function'){ + throw new CsvError('CSV_INVALID_OPTION_ON_RECORD', [ + 'Invalid option `on_record`:', + 'expect a function,', + `got ${JSON.stringify(options.on_record)}` + ], options); + } + // Normalize option `quote` + if(options.quote === null || options.quote === false || options.quote === ''){ + options.quote = null; + }else { + if(options.quote === undefined || options.quote === true){ + options.quote = Buffer.from('"', options.encoding); + }else if(typeof options.quote === 'string'){ + options.quote = Buffer.from(options.quote, options.encoding); + } + if(!Buffer.isBuffer(options.quote)){ + throw new Error(`Invalid Option: quote must be a buffer or a string, got ${JSON.stringify(options.quote)}`); } - // Normalize option `record_delimiter` - if(options.record_delimiter === undefined){ - options.record_delimiter = []; - }else if(typeof options.record_delimiter === 'string' || Buffer.isBuffer(options.record_delimiter)){ - if(options.record_delimiter.length === 0){ - throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ - 'Invalid option `record_delimiter`:', - 'value must be a non empty string or buffer,', - `got ${JSON.stringify(options.record_delimiter)}` - ], options); - } - options.record_delimiter = [options.record_delimiter]; - }else if(!Array.isArray(options.record_delimiter)){ + } + // Normalize option `raw` + if(options.raw === undefined || options.raw === null || options.raw === false){ + options.raw = false; + }else if(options.raw !== true){ + throw new Error(`Invalid Option: raw must be true, got ${JSON.stringify(options.raw)}`); + } + // Normalize option `record_delimiter` + if(options.record_delimiter === undefined){ + options.record_delimiter = []; + }else if(typeof options.record_delimiter === 'string' || Buffer.isBuffer(options.record_delimiter)){ + if(options.record_delimiter.length === 0){ throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ 'Invalid option `record_delimiter`:', - 'value must be a string, a buffer or array of string|buffer,', + 'value must be a non empty string or buffer,', `got ${JSON.stringify(options.record_delimiter)}` ], options); } - options.record_delimiter = options.record_delimiter.map(function(rd, i){ - if(typeof rd !== 'string' && ! Buffer.isBuffer(rd)){ - throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ - 'Invalid option `record_delimiter`:', - 'value must be a string, a buffer or array of string|buffer', - `at index ${i},`, - `got ${JSON.stringify(rd)}` - ], options); - }else if(rd.length === 0){ - throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ - 'Invalid option `record_delimiter`:', - 'value must be a non empty string or buffer', - `at index ${i},`, - `got ${JSON.stringify(rd)}` - ], options); - } - if(typeof rd === 'string'){ - rd = Buffer.from(rd, options.encoding); - } - return rd; - }); - // Normalize option `relax_column_count` - if(typeof options.relax_column_count === 'boolean');else if(options.relax_column_count === undefined || options.relax_column_count === null){ - options.relax_column_count = false; - }else { - throw new Error(`Invalid Option: relax_column_count must be a boolean, got ${JSON.stringify(options.relax_column_count)}`); - } - if(typeof options.relax_column_count_less === 'boolean');else if(options.relax_column_count_less === undefined || options.relax_column_count_less === null){ - options.relax_column_count_less = false; - }else { - throw new Error(`Invalid Option: relax_column_count_less must be a boolean, got ${JSON.stringify(options.relax_column_count_less)}`); - } - if(typeof options.relax_column_count_more === 'boolean');else if(options.relax_column_count_more === undefined || options.relax_column_count_more === null){ - options.relax_column_count_more = false; - }else { - throw new Error(`Invalid Option: relax_column_count_more must be a boolean, got ${JSON.stringify(options.relax_column_count_more)}`); - } - // Normalize option `relax_quotes` - if(typeof options.relax_quotes === 'boolean');else if(options.relax_quotes === undefined || options.relax_quotes === null){ - options.relax_quotes = false; - }else { - throw new Error(`Invalid Option: relax_quotes must be a boolean, got ${JSON.stringify(options.relax_quotes)}`); - } - // Normalize option `skip_empty_lines` - if(typeof options.skip_empty_lines === 'boolean');else if(options.skip_empty_lines === undefined || options.skip_empty_lines === null){ - options.skip_empty_lines = false; - }else { - throw new Error(`Invalid Option: skip_empty_lines must be a boolean, got ${JSON.stringify(options.skip_empty_lines)}`); - } - // Normalize option `skip_records_with_empty_values` - if(typeof options.skip_records_with_empty_values === 'boolean');else if(options.skip_records_with_empty_values === undefined || options.skip_records_with_empty_values === null){ - options.skip_records_with_empty_values = false; - }else { - throw new Error(`Invalid Option: skip_records_with_empty_values must be a boolean, got ${JSON.stringify(options.skip_records_with_empty_values)}`); - } - // Normalize option `skip_records_with_error` - if(typeof options.skip_records_with_error === 'boolean');else if(options.skip_records_with_error === undefined || options.skip_records_with_error === null){ - options.skip_records_with_error = false; - }else { - throw new Error(`Invalid Option: skip_records_with_error must be a boolean, got ${JSON.stringify(options.skip_records_with_error)}`); - } - // Normalize option `rtrim` - if(options.rtrim === undefined || options.rtrim === null || options.rtrim === false){ - options.rtrim = false; - }else if(options.rtrim !== true){ - throw new Error(`Invalid Option: rtrim must be a boolean, got ${JSON.stringify(options.rtrim)}`); - } - // Normalize option `ltrim` - if(options.ltrim === undefined || options.ltrim === null || options.ltrim === false){ - options.ltrim = false; - }else if(options.ltrim !== true){ - throw new Error(`Invalid Option: ltrim must be a boolean, got ${JSON.stringify(options.ltrim)}`); - } - // Normalize option `trim` - if(options.trim === undefined || options.trim === null || options.trim === false){ - options.trim = false; - }else if(options.trim !== true){ - throw new Error(`Invalid Option: trim must be a boolean, got ${JSON.stringify(options.trim)}`); - } - // Normalize options `trim`, `ltrim` and `rtrim` - if(options.trim === true && opts.ltrim !== false){ - options.ltrim = true; - }else if(options.ltrim !== true){ - options.ltrim = false; - } - if(options.trim === true && opts.rtrim !== false){ - options.rtrim = true; - }else if(options.rtrim !== true){ - options.rtrim = false; - } - // Normalize option `to` - if(options.to === undefined || options.to === null){ - options.to = -1; - }else { - if(typeof options.to === 'string' && /\d+/.test(options.to)){ - options.to = parseInt(options.to); - } - if(Number.isInteger(options.to)){ - if(options.to <= 0){ - throw new Error(`Invalid Option: to must be a positive integer greater than 0, got ${JSON.stringify(opts.to)}`); - } - }else { - throw new Error(`Invalid Option: to must be an integer, got ${JSON.stringify(opts.to)}`); - } - } - // Normalize option `to_line` - if(options.to_line === undefined || options.to_line === null){ - options.to_line = -1; - }else { - if(typeof options.to_line === 'string' && /\d+/.test(options.to_line)){ - options.to_line = parseInt(options.to_line); - } - if(Number.isInteger(options.to_line)){ - if(options.to_line <= 0){ - throw new Error(`Invalid Option: to_line must be a positive integer greater than 0, got ${JSON.stringify(opts.to_line)}`); - } - }else { - throw new Error(`Invalid Option: to_line must be an integer, got ${JSON.stringify(opts.to_line)}`); - } - } - this.info = { - bytes: 0, - comment_lines: 0, - empty_lines: 0, - invalid_field_length: 0, - lines: 1, - records: 0 - }; - this.options = options; - this.state = { - bomSkipped: false, - bufBytesStart: 0, - castField: fnCastField, - commenting: false, - // Current error encountered by a record - error: undefined, - enabled: options.from_line === 1, - escaping: false, - escapeIsQuote: Buffer.isBuffer(options.escape) && Buffer.isBuffer(options.quote) && Buffer.compare(options.escape, options.quote) === 0, - // columns can be `false`, `true`, `Array` - expectedRecordLength: Array.isArray(options.columns) ? options.columns.length : undefined, - field: new ResizeableBuffer(20), - firstLineToHeaders: fnFirstLineToHeaders, - needMoreDataSize: Math.max( - // Skip if the remaining buffer smaller than comment - options.comment !== null ? options.comment.length : 0, - // Skip if the remaining buffer can be delimiter - ...options.delimiter.map((delimiter) => delimiter.length), - // Skip if the remaining buffer can be escape sequence - options.quote !== null ? options.quote.length : 0, - ), - previousBuf: undefined, - quoting: false, - stop: false, - rawBuffer: new ResizeableBuffer(100), - record: [], - recordHasError: false, - record_length: 0, - recordDelimiterMaxLength: options.record_delimiter.length === 0 ? 2 : Math.max(...options.record_delimiter.map((v) => v.length)), - trimChars: [Buffer.from(' ', options.encoding)[0], Buffer.from('\t', options.encoding)[0]], - wasQuoting: false, - wasRowDelimiter: false - }; + options.record_delimiter = [options.record_delimiter]; + }else if(!Array.isArray(options.record_delimiter)){ + throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ + 'Invalid option `record_delimiter`:', + 'value must be a string, a buffer or array of string|buffer,', + `got ${JSON.stringify(options.record_delimiter)}` + ], options); } - // Implementation of `Transform._transform` - _transform(buf, encoding, callback){ - if(this.state.stop === true){ - return; + options.record_delimiter = options.record_delimiter.map(function(rd, i){ + if(typeof rd !== 'string' && ! Buffer.isBuffer(rd)){ + throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ + 'Invalid option `record_delimiter`:', + 'value must be a string, a buffer or array of string|buffer', + `at index ${i},`, + `got ${JSON.stringify(rd)}` + ], options); + }else if(rd.length === 0){ + throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ + 'Invalid option `record_delimiter`:', + 'value must be a non empty string or buffer', + `at index ${i},`, + `got ${JSON.stringify(rd)}` + ], options); } - const err = this.__parse(buf, false); - if(err !== undefined){ - this.state.stop = true; + if(typeof rd === 'string'){ + rd = Buffer.from(rd, options.encoding); } - callback(err); + return rd; + }); + // Normalize option `relax_column_count` + if(typeof options.relax_column_count === 'boolean');else if(options.relax_column_count === undefined || options.relax_column_count === null){ + options.relax_column_count = false; + }else { + throw new Error(`Invalid Option: relax_column_count must be a boolean, got ${JSON.stringify(options.relax_column_count)}`); } - // Implementation of `Transform._flush` - _flush(callback){ - if(this.state.stop === true){ - return; + if(typeof options.relax_column_count_less === 'boolean');else if(options.relax_column_count_less === undefined || options.relax_column_count_less === null){ + options.relax_column_count_less = false; + }else { + throw new Error(`Invalid Option: relax_column_count_less must be a boolean, got ${JSON.stringify(options.relax_column_count_less)}`); + } + if(typeof options.relax_column_count_more === 'boolean');else if(options.relax_column_count_more === undefined || options.relax_column_count_more === null){ + options.relax_column_count_more = false; + }else { + throw new Error(`Invalid Option: relax_column_count_more must be a boolean, got ${JSON.stringify(options.relax_column_count_more)}`); + } + // Normalize option `relax_quotes` + if(typeof options.relax_quotes === 'boolean');else if(options.relax_quotes === undefined || options.relax_quotes === null){ + options.relax_quotes = false; + }else { + throw new Error(`Invalid Option: relax_quotes must be a boolean, got ${JSON.stringify(options.relax_quotes)}`); + } + // Normalize option `skip_empty_lines` + if(typeof options.skip_empty_lines === 'boolean');else if(options.skip_empty_lines === undefined || options.skip_empty_lines === null){ + options.skip_empty_lines = false; + }else { + throw new Error(`Invalid Option: skip_empty_lines must be a boolean, got ${JSON.stringify(options.skip_empty_lines)}`); + } + // Normalize option `skip_records_with_empty_values` + if(typeof options.skip_records_with_empty_values === 'boolean');else if(options.skip_records_with_empty_values === undefined || options.skip_records_with_empty_values === null){ + options.skip_records_with_empty_values = false; + }else { + throw new Error(`Invalid Option: skip_records_with_empty_values must be a boolean, got ${JSON.stringify(options.skip_records_with_empty_values)}`); + } + // Normalize option `skip_records_with_error` + if(typeof options.skip_records_with_error === 'boolean');else if(options.skip_records_with_error === undefined || options.skip_records_with_error === null){ + options.skip_records_with_error = false; + }else { + throw new Error(`Invalid Option: skip_records_with_error must be a boolean, got ${JSON.stringify(options.skip_records_with_error)}`); + } + // Normalize option `rtrim` + if(options.rtrim === undefined || options.rtrim === null || options.rtrim === false){ + options.rtrim = false; + }else if(options.rtrim !== true){ + throw new Error(`Invalid Option: rtrim must be a boolean, got ${JSON.stringify(options.rtrim)}`); + } + // Normalize option `ltrim` + if(options.ltrim === undefined || options.ltrim === null || options.ltrim === false){ + options.ltrim = false; + }else if(options.ltrim !== true){ + throw new Error(`Invalid Option: ltrim must be a boolean, got ${JSON.stringify(options.ltrim)}`); + } + // Normalize option `trim` + if(options.trim === undefined || options.trim === null || options.trim === false){ + options.trim = false; + }else if(options.trim !== true){ + throw new Error(`Invalid Option: trim must be a boolean, got ${JSON.stringify(options.trim)}`); + } + // Normalize options `trim`, `ltrim` and `rtrim` + if(options.trim === true && opts.ltrim !== false){ + options.ltrim = true; + }else if(options.ltrim !== true){ + options.ltrim = false; + } + if(options.trim === true && opts.rtrim !== false){ + options.rtrim = true; + }else if(options.rtrim !== true){ + options.rtrim = false; + } + // Normalize option `to` + if(options.to === undefined || options.to === null){ + options.to = -1; + }else { + if(typeof options.to === 'string' && /\d+/.test(options.to)){ + options.to = parseInt(options.to); + } + if(Number.isInteger(options.to)){ + if(options.to <= 0){ + throw new Error(`Invalid Option: to must be a positive integer greater than 0, got ${JSON.stringify(opts.to)}`); + } + }else { + throw new Error(`Invalid Option: to must be an integer, got ${JSON.stringify(opts.to)}`); } - const err = this.__parse(undefined, true); - callback(err); } - // Central parser implementation - __parse(nextBuf, end){ - const {bom, comment, escape, from_line, ltrim, max_record_size, quote, raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options; - let {record_delimiter} = this.options; - const {bomSkipped, previousBuf, rawBuffer, escapeIsQuote} = this.state; - let buf; - if(previousBuf === undefined){ - if(nextBuf === undefined){ - // Handle empty string - this.push(null); - return; - }else { - buf = nextBuf; + // Normalize option `to_line` + if(options.to_line === undefined || options.to_line === null){ + options.to_line = -1; + }else { + if(typeof options.to_line === 'string' && /\d+/.test(options.to_line)){ + options.to_line = parseInt(options.to_line); + } + if(Number.isInteger(options.to_line)){ + if(options.to_line <= 0){ + throw new Error(`Invalid Option: to_line must be a positive integer greater than 0, got ${JSON.stringify(opts.to_line)}`); } - }else if(previousBuf !== undefined && nextBuf === undefined){ - buf = previousBuf; }else { - buf = Buffer.concat([previousBuf, nextBuf]); + throw new Error(`Invalid Option: to_line must be an integer, got ${JSON.stringify(opts.to_line)}`); } - // Handle UTF BOM - if(bomSkipped === false){ - if(bom === false){ - this.state.bomSkipped = true; - }else if(buf.length < 3){ - // No enough data - if(end === false){ - // Wait for more data - this.state.previousBuf = buf; + } + return options; +}; + +const isRecordEmpty = function(record){ + return record.every((field) => field == null || field.toString && field.toString().trim() === ''); +}; + +// white space characters +// https://en.wikipedia.org/wiki/Whitespace_character +// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions/Character_Classes#Types +// \f\n\r\t\v\u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff +const tab = 9; +const nl = 10; // \n, 0x0A in hexadecimal, 10 in decimal +const np = 12; +const cr = 13; // \r, 0x0D in hexadécimal, 13 in decimal +const space = 32; +const boms = { + // Note, the following are equals: + // Buffer.from("\ufeff") + // Buffer.from([239, 187, 191]) + // Buffer.from('EFBBBF', 'hex') + 'utf8': Buffer.from([239, 187, 191]), + // Note, the following are equals: + // Buffer.from "\ufeff", 'utf16le + // Buffer.from([255, 254]) + 'utf16le': Buffer.from([255, 254]) +}; + +const transform = function(original_options = {}) { + const info = { + bytes: 0, + comment_lines: 0, + empty_lines: 0, + invalid_field_length: 0, + lines: 1, + records: 0 + }; + const options = normalize_options(original_options); + return { + info: info, + original_options: original_options, + options: options, + state: init_state(options), + __needMoreData: function(i, bufLen, end){ + if(end) return false; + const {quote} = this.options; + const {quoting, needMoreDataSize, recordDelimiterMaxLength} = this.state; + const numOfCharLeft = bufLen - i - 1; + const requiredLength = Math.max( + needMoreDataSize, + // Skip if the remaining buffer smaller than record delimiter + recordDelimiterMaxLength, + // Skip if the remaining buffer can be record delimiter following the closing quote + // 1 is for quote.length + quoting ? (quote.length + recordDelimiterMaxLength) : 0, + ); + return numOfCharLeft < requiredLength; + }, + // Central parser implementation + parse: function(nextBuf, end, push, close){ + const {bom, comment, escape, from_line, ltrim, max_record_size, quote, raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options; + let {record_delimiter} = this.options; + const {bomSkipped, previousBuf, rawBuffer, escapeIsQuote} = this.state; + let buf; + if(previousBuf === undefined){ + if(nextBuf === undefined){ + // Handle empty string + close(); return; + }else { + buf = nextBuf; } + }else if(previousBuf !== undefined && nextBuf === undefined){ + buf = previousBuf; }else { - for(const encoding in boms){ - if(boms[encoding].compare(buf, 0, boms[encoding].length) === 0){ - // Skip BOM - const bomLength = boms[encoding].length; - this.state.bufBytesStart += bomLength; - buf = buf.slice(bomLength); - // Renormalize original options with the new encoding - this.__normalizeOptions({...this.__originalOptions, encoding: encoding}); - break; + buf = Buffer.concat([previousBuf, nextBuf]); + } + // Handle UTF BOM + if(bomSkipped === false){ + if(bom === false){ + this.state.bomSkipped = true; + }else if(buf.length < 3){ + // No enough data + if(end === false){ + // Wait for more data + this.state.previousBuf = buf; + return; } + }else { + for(const encoding in boms){ + if(boms[encoding].compare(buf, 0, boms[encoding].length) === 0){ + // Skip BOM + const bomLength = boms[encoding].length; + this.state.bufBytesStart += bomLength; + buf = buf.slice(bomLength); + // Renormalize original options with the new encoding + this.options = normalize_options({...this.original_options, encoding: encoding}); + break; + } + } + this.state.bomSkipped = true; } - this.state.bomSkipped = true; - } - } - const bufLen = buf.length; - let pos; - for(pos = 0; pos < bufLen; pos++){ - // Ensure we get enough space to look ahead - // There should be a way to move this out of the loop - if(this.__needMoreData(pos, bufLen, end)){ - break; } - if(this.state.wasRowDelimiter === true){ - this.info.lines++; - this.state.wasRowDelimiter = false; - } - if(to_line !== -1 && this.info.lines > to_line){ - this.state.stop = true; - this.push(null); - return; - } - // Auto discovery of record_delimiter, unix, mac and windows supported - if(this.state.quoting === false && record_delimiter.length === 0){ - const record_delimiterCount = this.__autoDiscoverRecordDelimiter(buf, pos); - if(record_delimiterCount){ - record_delimiter = this.options.record_delimiter; + const bufLen = buf.length; + let pos; + for(pos = 0; pos < bufLen; pos++){ + // Ensure we get enough space to look ahead + // There should be a way to move this out of the loop + if(this.__needMoreData(pos, bufLen, end)){ + break; } - } - const chr = buf[pos]; - if(raw === true){ - rawBuffer.append(chr); - } - if((chr === cr || chr === nl) && this.state.wasRowDelimiter === false){ - this.state.wasRowDelimiter = true; - } - // Previous char was a valid escape char - // treat the current char as a regular char - if(this.state.escaping === true){ - this.state.escaping = false; - }else { - // Escape is only active inside quoted fields - // We are quoting, the char is an escape chr and there is a chr to escape - // if(escape !== null && this.state.quoting === true && chr === escape && pos + 1 < bufLen){ - if(escape !== null && this.state.quoting === true && this.__isEscape(buf, pos, chr) && pos + escape.length < bufLen){ - if(escapeIsQuote){ - if(this.__isQuote(buf, pos+escape.length)){ - this.state.escaping = true; - pos += escape.length - 1; - continue; - } - }else { - this.state.escaping = true; - pos += escape.length - 1; - continue; + if(this.state.wasRowDelimiter === true){ + this.info.lines++; + this.state.wasRowDelimiter = false; + } + if(to_line !== -1 && this.info.lines > to_line){ + this.state.stop = true; + close(); + return; + } + // Auto discovery of record_delimiter, unix, mac and windows supported + if(this.state.quoting === false && record_delimiter.length === 0){ + const record_delimiterCount = this.__autoDiscoverRecordDelimiter(buf, pos); + if(record_delimiterCount){ + record_delimiter = this.options.record_delimiter; } } - // Not currently escaping and chr is a quote - // TODO: need to compare bytes instead of single char - if(this.state.commenting === false && this.__isQuote(buf, pos)){ - if(this.state.quoting === true){ - const nextChr = buf[pos+quote.length]; - const isNextChrTrimable = rtrim && this.__isCharTrimable(nextChr); - const isNextChrComment = comment !== null && this.__compareBytes(comment, buf, pos+quote.length, nextChr); - const isNextChrDelimiter = this.__isDelimiter(buf, pos+quote.length, nextChr); - const isNextChrRecordDelimiter = record_delimiter.length === 0 ? this.__autoDiscoverRecordDelimiter(buf, pos+quote.length) : this.__isRecordDelimiter(nextChr, buf, pos+quote.length); - // Escape a quote - // Treat next char as a regular character - if(escape !== null && this.__isEscape(buf, pos, chr) && this.__isQuote(buf, pos + escape.length)){ + const chr = buf[pos]; + if(raw === true){ + rawBuffer.append(chr); + } + if((chr === cr || chr === nl) && this.state.wasRowDelimiter === false){ + this.state.wasRowDelimiter = true; + } + // Previous char was a valid escape char + // treat the current char as a regular char + if(this.state.escaping === true){ + this.state.escaping = false; + }else { + // Escape is only active inside quoted fields + // We are quoting, the char is an escape chr and there is a chr to escape + // if(escape !== null && this.state.quoting === true && chr === escape && pos + 1 < bufLen){ + if(escape !== null && this.state.quoting === true && this.__isEscape(buf, pos, chr) && pos + escape.length < bufLen){ + if(escapeIsQuote){ + if(this.__isQuote(buf, pos+escape.length)){ + this.state.escaping = true; + pos += escape.length - 1; + continue; + } + }else { + this.state.escaping = true; pos += escape.length - 1; - }else if(!nextChr || isNextChrDelimiter || isNextChrRecordDelimiter || isNextChrComment || isNextChrTrimable){ - this.state.quoting = false; - this.state.wasQuoting = true; - pos += quote.length - 1; continue; - }else if(relax_quotes === false){ - const err = this.__error( - new CsvError('CSV_INVALID_CLOSING_QUOTE', [ - 'Invalid Closing Quote:', - `got "${String.fromCharCode(nextChr)}"`, - `at line ${this.info.lines}`, - 'instead of delimiter, record delimiter, trimable character', - '(if activated) or comment', - ], this.options, this.__infoField()) - ); - if(err !== undefined) return err; - }else { - this.state.quoting = false; - this.state.wasQuoting = true; - this.state.field.prepend(quote); - pos += quote.length - 1; } - }else { - if(this.state.field.length !== 0){ - // In relax_quotes mode, treat opening quote preceded by chrs as regular - if(relax_quotes === false){ + } + // Not currently escaping and chr is a quote + // TODO: need to compare bytes instead of single char + if(this.state.commenting === false && this.__isQuote(buf, pos)){ + if(this.state.quoting === true){ + const nextChr = buf[pos+quote.length]; + const isNextChrTrimable = rtrim && this.__isCharTrimable(nextChr); + const isNextChrComment = comment !== null && this.__compareBytes(comment, buf, pos+quote.length, nextChr); + const isNextChrDelimiter = this.__isDelimiter(buf, pos+quote.length, nextChr); + const isNextChrRecordDelimiter = record_delimiter.length === 0 ? this.__autoDiscoverRecordDelimiter(buf, pos+quote.length) : this.__isRecordDelimiter(nextChr, buf, pos+quote.length); + // Escape a quote + // Treat next char as a regular character + if(escape !== null && this.__isEscape(buf, pos, chr) && this.__isQuote(buf, pos + escape.length)){ + pos += escape.length - 1; + }else if(!nextChr || isNextChrDelimiter || isNextChrRecordDelimiter || isNextChrComment || isNextChrTrimable){ + this.state.quoting = false; + this.state.wasQuoting = true; + pos += quote.length - 1; + continue; + }else if(relax_quotes === false){ const err = this.__error( - new CsvError('INVALID_OPENING_QUOTE', [ - 'Invalid Opening Quote:', - `a quote is found inside a field at line ${this.info.lines}`, - ], this.options, this.__infoField(), { - field: this.state.field, - }) + new CsvError('CSV_INVALID_CLOSING_QUOTE', [ + 'Invalid Closing Quote:', + `got "${String.fromCharCode(nextChr)}"`, + `at line ${this.info.lines}`, + 'instead of delimiter, record delimiter, trimable character', + '(if activated) or comment', + ], this.options, this.__infoField()) ); if(err !== undefined) return err; + }else { + this.state.quoting = false; + this.state.wasQuoting = true; + this.state.field.prepend(quote); + pos += quote.length - 1; } }else { - this.state.quoting = true; - pos += quote.length - 1; - continue; - } - } - } - if(this.state.quoting === false){ - const recordDelimiterLength = this.__isRecordDelimiter(chr, buf, pos); - if(recordDelimiterLength !== 0){ - // Do not emit comments which take a full line - const skipCommentLine = this.state.commenting && (this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0); - if(skipCommentLine){ - this.info.comment_lines++; - // Skip full comment line - }else { - // Activate records emition if above from_line - if(this.state.enabled === false && this.info.lines + (this.state.wasRowDelimiter === true ? 1: 0) >= from_line){ - this.state.enabled = true; - this.__resetField(); - this.__resetRecord(); - pos += recordDelimiterLength - 1; + if(this.state.field.length !== 0){ + // In relax_quotes mode, treat opening quote preceded by chrs as regular + if(relax_quotes === false){ + const err = this.__error( + new CsvError('INVALID_OPENING_QUOTE', [ + 'Invalid Opening Quote:', + `a quote is found inside a field at line ${this.info.lines}`, + ], this.options, this.__infoField(), { + field: this.state.field, + }) + ); + if(err !== undefined) return err; + } + }else { + this.state.quoting = true; + pos += quote.length - 1; continue; } - // Skip if line is empty and skip_empty_lines activated - if(skip_empty_lines === true && this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0){ - this.info.empty_lines++; - pos += recordDelimiterLength - 1; - continue; + } + } + if(this.state.quoting === false){ + const recordDelimiterLength = this.__isRecordDelimiter(chr, buf, pos); + if(recordDelimiterLength !== 0){ + // Do not emit comments which take a full line + const skipCommentLine = this.state.commenting && (this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0); + if(skipCommentLine){ + this.info.comment_lines++; + // Skip full comment line + }else { + // Activate records emition if above from_line + if(this.state.enabled === false && this.info.lines + (this.state.wasRowDelimiter === true ? 1: 0) >= from_line){ + this.state.enabled = true; + this.__resetField(); + this.__resetRecord(); + pos += recordDelimiterLength - 1; + continue; + } + // Skip if line is empty and skip_empty_lines activated + if(skip_empty_lines === true && this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0){ + this.info.empty_lines++; + pos += recordDelimiterLength - 1; + continue; + } + this.info.bytes = this.state.bufBytesStart + pos; + const errField = this.__onField(); + if(errField !== undefined) return errField; + this.info.bytes = this.state.bufBytesStart + pos + recordDelimiterLength; + const errRecord = this.__onRecord(push); + if(errRecord !== undefined) return errRecord; + if(to !== -1 && this.info.records >= to){ + this.state.stop = true; + close(); + return; + } } + this.state.commenting = false; + pos += recordDelimiterLength - 1; + continue; + } + if(this.state.commenting){ + continue; + } + const commentCount = comment === null ? 0 : this.__compareBytes(comment, buf, pos, chr); + if(commentCount !== 0){ + this.state.commenting = true; + continue; + } + const delimiterLength = this.__isDelimiter(buf, pos, chr); + if(delimiterLength !== 0){ this.info.bytes = this.state.bufBytesStart + pos; const errField = this.__onField(); if(errField !== undefined) return errField; - this.info.bytes = this.state.bufBytesStart + pos + recordDelimiterLength; - const errRecord = this.__onRecord(); - if(errRecord !== undefined) return errRecord; - if(to !== -1 && this.info.records >= to){ - this.state.stop = true; - this.push(null); - return; - } + pos += delimiterLength - 1; + continue; } - this.state.commenting = false; - pos += recordDelimiterLength - 1; - continue; - } - if(this.state.commenting){ - continue; - } - const commentCount = comment === null ? 0 : this.__compareBytes(comment, buf, pos, chr); - if(commentCount !== 0){ - this.state.commenting = true; - continue; } - const delimiterLength = this.__isDelimiter(buf, pos, chr); - if(delimiterLength !== 0){ - this.info.bytes = this.state.bufBytesStart + pos; - const errField = this.__onField(); - if(errField !== undefined) return errField; - pos += delimiterLength - 1; - continue; + } + if(this.state.commenting === false){ + if(max_record_size !== 0 && this.state.record_length + this.state.field.length > max_record_size){ + const err = this.__error( + new CsvError('CSV_MAX_RECORD_SIZE', [ + 'Max Record Size:', + 'record exceed the maximum number of tolerated bytes', + `of ${max_record_size}`, + `at line ${this.info.lines}`, + ], this.options, this.__infoField()) + ); + if(err !== undefined) return err; } } - } - if(this.state.commenting === false){ - if(max_record_size !== 0 && this.state.record_length + this.state.field.length > max_record_size){ + const lappend = ltrim === false || this.state.quoting === true || this.state.field.length !== 0 || !this.__isCharTrimable(chr); + // rtrim in non quoting is handle in __onField + const rappend = rtrim === false || this.state.wasQuoting === false; + if(lappend === true && rappend === true){ + this.state.field.append(chr); + }else if(rtrim === true && !this.__isCharTrimable(chr)){ const err = this.__error( - new CsvError('CSV_MAX_RECORD_SIZE', [ - 'Max Record Size:', - 'record exceed the maximum number of tolerated bytes', - `of ${max_record_size}`, + new CsvError('CSV_NON_TRIMABLE_CHAR_AFTER_CLOSING_QUOTE', [ + 'Invalid Closing Quote:', + 'found non trimable byte after quote', `at line ${this.info.lines}`, ], this.options, this.__infoField()) ); if(err !== undefined) return err; } } - const lappend = ltrim === false || this.state.quoting === true || this.state.field.length !== 0 || !this.__isCharTrimable(chr); - // rtrim in non quoting is handle in __onField - const rappend = rtrim === false || this.state.wasQuoting === false; - if(lappend === true && rappend === true){ - this.state.field.append(chr); - }else if(rtrim === true && !this.__isCharTrimable(chr)){ - const err = this.__error( - new CsvError('CSV_NON_TRIMABLE_CHAR_AFTER_CLOSING_QUOTE', [ - 'Invalid Closing Quote:', - 'found non trimable byte after quote', - `at line ${this.info.lines}`, - ], this.options, this.__infoField()) - ); - if(err !== undefined) return err; - } - } - if(end === true){ - // Ensure we are not ending in a quoting state - if(this.state.quoting === true){ - const err = this.__error( - new CsvError('CSV_QUOTE_NOT_CLOSED', [ - 'Quote Not Closed:', - `the parsing is finished with an opening quote at line ${this.info.lines}`, - ], this.options, this.__infoField()) - ); - if(err !== undefined) return err; + if(end === true){ + // Ensure we are not ending in a quoting state + if(this.state.quoting === true){ + const err = this.__error( + new CsvError('CSV_QUOTE_NOT_CLOSED', [ + 'Quote Not Closed:', + `the parsing is finished with an opening quote at line ${this.info.lines}`, + ], this.options, this.__infoField()) + ); + if(err !== undefined) return err; + }else { + // Skip last line if it has no characters + if(this.state.wasQuoting === true || this.state.record.length !== 0 || this.state.field.length !== 0){ + this.info.bytes = this.state.bufBytesStart + pos; + const errField = this.__onField(); + if(errField !== undefined) return errField; + const errRecord = this.__onRecord(push); + if(errRecord !== undefined) return errRecord; + }else if(this.state.wasRowDelimiter === true){ + this.info.empty_lines++; + }else if(this.state.commenting === true){ + this.info.comment_lines++; + } + } }else { - // Skip last line if it has no characters - if(this.state.wasQuoting === true || this.state.record.length !== 0 || this.state.field.length !== 0){ - this.info.bytes = this.state.bufBytesStart + pos; - const errField = this.__onField(); - if(errField !== undefined) return errField; - const errRecord = this.__onRecord(); - if(errRecord !== undefined) return errRecord; - }else if(this.state.wasRowDelimiter === true){ - this.info.empty_lines++; - }else if(this.state.commenting === true){ - this.info.comment_lines++; + this.state.bufBytesStart += pos; + this.state.previousBuf = buf.slice(pos); + } + if(this.state.wasRowDelimiter === true){ + this.info.lines++; + this.state.wasRowDelimiter = false; + } + }, + __onRecord: function(push){ + const {columns, group_columns_by_name, encoding, info, from, relax_column_count, relax_column_count_less, relax_column_count_more, raw, skip_records_with_empty_values} = this.options; + const {enabled, record} = this.state; + if(enabled === false){ + return this.__resetRecord(); + } + // Convert the first line into column names + const recordLength = record.length; + if(columns === true){ + if(skip_records_with_empty_values === true && isRecordEmpty(record)){ + this.__resetRecord(); + return; + } + return this.__firstLineToColumns(record); + } + if(columns === false && this.info.records === 0){ + this.state.expectedRecordLength = recordLength; + } + if(recordLength !== this.state.expectedRecordLength){ + const err = columns === false ? + new CsvError('CSV_RECORD_INCONSISTENT_FIELDS_LENGTH', [ + 'Invalid Record Length:', + `expect ${this.state.expectedRecordLength},`, + `got ${recordLength} on line ${this.info.lines}`, + ], this.options, this.__infoField(), { + record: record, + }) + : + new CsvError('CSV_RECORD_INCONSISTENT_COLUMNS', [ + 'Invalid Record Length:', + `columns length is ${columns.length},`, // rename columns + `got ${recordLength} on line ${this.info.lines}`, + ], this.options, this.__infoField(), { + record: record, + }); + if(relax_column_count === true || + (relax_column_count_less === true && recordLength < this.state.expectedRecordLength) || + (relax_column_count_more === true && recordLength > this.state.expectedRecordLength)){ + this.info.invalid_field_length++; + this.state.error = err; + // Error is undefined with skip_records_with_error + }else { + const finalErr = this.__error(err); + if(finalErr) return finalErr; } } - }else { - this.state.bufBytesStart += pos; - this.state.previousBuf = buf.slice(pos); - } - if(this.state.wasRowDelimiter === true){ - this.info.lines++; - this.state.wasRowDelimiter = false; - } - } - __onRecord(){ - const {columns, group_columns_by_name, encoding, info, from, relax_column_count, relax_column_count_less, relax_column_count_more, raw, skip_records_with_empty_values} = this.options; - const {enabled, record} = this.state; - if(enabled === false){ - return this.__resetRecord(); - } - // Convert the first line into column names - const recordLength = record.length; - if(columns === true){ if(skip_records_with_empty_values === true && isRecordEmpty(record)){ this.__resetRecord(); return; } - return this.__firstLineToColumns(record); - } - if(columns === false && this.info.records === 0){ - this.state.expectedRecordLength = recordLength; - } - if(recordLength !== this.state.expectedRecordLength){ - const err = columns === false ? - new CsvError('CSV_RECORD_INCONSISTENT_FIELDS_LENGTH', [ - 'Invalid Record Length:', - `expect ${this.state.expectedRecordLength},`, - `got ${recordLength} on line ${this.info.lines}`, - ], this.options, this.__infoField(), { - record: record, - }) - : - new CsvError('CSV_RECORD_INCONSISTENT_COLUMNS', [ - 'Invalid Record Length:', - `columns length is ${columns.length},`, // rename columns - `got ${recordLength} on line ${this.info.lines}`, - ], this.options, this.__infoField(), { - record: record, - }); - if(relax_column_count === true || - (relax_column_count_less === true && recordLength < this.state.expectedRecordLength) || - (relax_column_count_more === true && recordLength > this.state.expectedRecordLength)){ - this.info.invalid_field_length++; - this.state.error = err; - // Error is undefined with skip_records_with_error - }else { - const finalErr = this.__error(err); - if(finalErr) return finalErr; + if(this.state.recordHasError === true){ + this.__resetRecord(); + this.state.recordHasError = false; + return; } - } - if(skip_records_with_empty_values === true && isRecordEmpty(record)){ - this.__resetRecord(); - return; - } - if(this.state.recordHasError === true){ - this.__resetRecord(); - this.state.recordHasError = false; - return; - } - this.info.records++; - if(from === 1 || this.info.records >= from){ - const {objname} = this.options; - // With columns, records are object - if(columns !== false){ - const obj = {}; - // Transform record array to an object - for(let i = 0, l = record.length; i < l; i++){ - if(columns[i] === undefined || columns[i].disabled) continue; - // Turn duplicate columns into an array - if (group_columns_by_name === true && obj[columns[i].name] !== undefined) { - if (Array.isArray(obj[columns[i].name])) { - obj[columns[i].name] = obj[columns[i].name].concat(record[i]); + this.info.records++; + if(from === 1 || this.info.records >= from){ + const {objname} = this.options; + // With columns, records are object + if(columns !== false){ + const obj = {}; + // Transform record array to an object + for(let i = 0, l = record.length; i < l; i++){ + if(columns[i] === undefined || columns[i].disabled) continue; + // Turn duplicate columns into an array + if (group_columns_by_name === true && obj[columns[i].name] !== undefined) { + if (Array.isArray(obj[columns[i].name])) { + obj[columns[i].name] = obj[columns[i].name].concat(record[i]); + } else { + obj[columns[i].name] = [obj[columns[i].name], record[i]]; + } } else { - obj[columns[i].name] = [obj[columns[i].name], record[i]]; + obj[columns[i].name] = record[i]; } - } else { - obj[columns[i].name] = record[i]; - } - } - // Without objname (default) - if(raw === true || info === true){ - const extRecord = Object.assign( - {record: obj}, - (raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}), - (info === true ? {info: this.__infoRecord()}: {}) - ); - const err = this.__push( - objname === undefined ? extRecord : [obj[objname], extRecord] - ); - if(err){ - return err; - } - }else { - const err = this.__push( - objname === undefined ? obj : [obj[objname], obj] - ); - if(err){ - return err; } - } - // Without columns, records are array - }else { - if(raw === true || info === true){ - const extRecord = Object.assign( - {record: record}, - raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}, - info === true ? {info: this.__infoRecord()}: {} - ); - const err = this.__push( - objname === undefined ? extRecord : [record[objname], extRecord] - ); - if(err){ - return err; + // Without objname (default) + if(raw === true || info === true){ + const extRecord = Object.assign( + {record: obj}, + (raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}), + (info === true ? {info: this.__infoRecord()}: {}) + ); + const err = this.__push( + objname === undefined ? extRecord : [obj[objname], extRecord] + , push); + if(err){ + return err; + } + }else { + const err = this.__push( + objname === undefined ? obj : [obj[objname], obj] + , push); + if(err){ + return err; + } } + // Without columns, records are array }else { - const err = this.__push( - objname === undefined ? record : [record[objname], record] - ); - if(err){ - return err; + if(raw === true || info === true){ + const extRecord = Object.assign( + {record: record}, + raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}, + info === true ? {info: this.__infoRecord()}: {} + ); + const err = this.__push( + objname === undefined ? extRecord : [record[objname], extRecord] + , push); + if(err){ + return err; + } + }else { + const err = this.__push( + objname === undefined ? record : [record[objname], record] + , push); + if(err){ + return err; + } } } } - } - this.__resetRecord(); - } - __firstLineToColumns(record){ - const {firstLineToHeaders} = this.state; - try{ - const headers = firstLineToHeaders === undefined ? record : firstLineToHeaders.call(null, record); - if(!Array.isArray(headers)){ - return this.__error( - new CsvError('CSV_INVALID_COLUMN_MAPPING', [ - 'Invalid Column Mapping:', - 'expect an array from column function,', - `got ${JSON.stringify(headers)}` - ], this.options, this.__infoField(), { - headers: headers, - }) - ); - } - const normalizedHeaders = normalizeColumnsArray(headers); - this.state.expectedRecordLength = normalizedHeaders.length; - this.options.columns = normalizedHeaders; this.__resetRecord(); - return; - }catch(err){ - return err; - } - } - __resetRecord(){ - if(this.options.raw === true){ - this.state.rawBuffer.reset(); - } - this.state.error = undefined; - this.state.record = []; - this.state.record_length = 0; - } - __onField(){ - const {cast, encoding, rtrim, max_record_size} = this.options; - const {enabled, wasQuoting} = this.state; - // Short circuit for the from_line options - if(enabled === false){ - return this.__resetField(); - } - let field = this.state.field.toString(encoding); - if(rtrim === true && wasQuoting === false){ - field = field.trimRight(); - } - if(cast === true){ - const [err, f] = this.__cast(field); - if(err !== undefined) return err; - field = f; - } - this.state.record.push(field); - // Increment record length if record size must not exceed a limit - if(max_record_size !== 0 && typeof field === 'string'){ - this.state.record_length += field.length; - } - this.__resetField(); - } - __resetField(){ - this.state.field.reset(); - this.state.wasQuoting = false; - } - __push(record){ - const {on_record} = this.options; - if(on_record !== undefined){ - const info = this.__infoRecord(); + }, + __firstLineToColumns: function(record){ + const {firstLineToHeaders} = this.state; try{ - record = on_record.call(null, record, info); + const headers = firstLineToHeaders === undefined ? record : firstLineToHeaders.call(null, record); + if(!Array.isArray(headers)){ + return this.__error( + new CsvError('CSV_INVALID_COLUMN_MAPPING', [ + 'Invalid Column Mapping:', + 'expect an array from column function,', + `got ${JSON.stringify(headers)}` + ], this.options, this.__infoField(), { + headers: headers, + }) + ); + } + const normalizedHeaders = normalize_columns_array(headers); + this.state.expectedRecordLength = normalizedHeaders.length; + this.options.columns = normalizedHeaders; + this.__resetRecord(); + return; }catch(err){ return err; } - if(record === undefined || record === null){ return; } - } - this.push(record); - } - // Return a tuple with the error and the casted value - __cast(field){ - const {columns, relax_column_count} = this.options; - const isColumns = Array.isArray(columns); - // Dont loose time calling cast - // because the final record is an object - // and this field can't be associated to a key present in columns - if(isColumns === true && relax_column_count && this.options.columns.length <= this.state.record.length){ - return [undefined, undefined]; - } - if(this.state.castField !== null){ - try{ + }, + __resetRecord: function(){ + if(this.options.raw === true){ + this.state.rawBuffer.reset(); + } + this.state.error = undefined; + this.state.record = []; + this.state.record_length = 0; + }, + __onField: function(){ + const {cast, encoding, rtrim, max_record_size} = this.options; + const {enabled, wasQuoting} = this.state; + // Short circuit for the from_line options + if(enabled === false){ + return this.__resetField(); + } + let field = this.state.field.toString(encoding); + if(rtrim === true && wasQuoting === false){ + field = field.trimRight(); + } + if(cast === true){ + const [err, f] = this.__cast(field); + if(err !== undefined) return err; + field = f; + } + this.state.record.push(field); + // Increment record length if record size must not exceed a limit + if(max_record_size !== 0 && typeof field === 'string'){ + this.state.record_length += field.length; + } + this.__resetField(); + }, + __resetField: function(){ + this.state.field.reset(); + this.state.wasQuoting = false; + }, + __push: function(record, push){ + const {on_record} = this.options; + if(on_record !== undefined){ + const info = this.__infoRecord(); + try{ + record = on_record.call(null, record, info); + }catch(err){ + return err; + } + if(record === undefined || record === null){ return; } + } + push(record); + }, + // Return a tuple with the error and the casted value + __cast: function(field){ + const {columns, relax_column_count} = this.options; + const isColumns = Array.isArray(columns); + // Dont loose time calling cast + // because the final record is an object + // and this field can't be associated to a key present in columns + if(isColumns === true && relax_column_count && this.options.columns.length <= this.state.record.length){ + return [undefined, undefined]; + } + if(this.state.castField !== null){ + try{ + const info = this.__infoField(); + return [undefined, this.state.castField.call(null, field, info)]; + }catch(err){ + return [err]; + } + } + if(this.__isFloat(field)){ + return [undefined, parseFloat(field)]; + }else if(this.options.cast_date !== false){ const info = this.__infoField(); - return [undefined, this.state.castField.call(null, field, info)]; - }catch(err){ - return [err]; + return [undefined, this.options.cast_date.call(null, field, info)]; } - } - if(this.__isFloat(field)){ - return [undefined, parseFloat(field)]; - }else if(this.options.cast_date !== false){ - const info = this.__infoField(); - return [undefined, this.options.cast_date.call(null, field, info)]; - } - return [undefined, field]; - } - // Helper to test if a character is a space or a line delimiter - __isCharTrimable(chr){ - return chr === space || chr === tab || chr === cr || chr === nl || chr === np; - } - // Keep it in case we implement the `cast_int` option - // __isInt(value){ - // // return Number.isInteger(parseInt(value)) - // // return !isNaN( parseInt( obj ) ); - // return /^(\-|\+)?[1-9][0-9]*$/.test(value) - // } - __isFloat(value){ - return (value - parseFloat(value) + 1) >= 0; // Borrowed from jquery - } - __compareBytes(sourceBuf, targetBuf, targetPos, firstByte){ - if(sourceBuf[0] !== firstByte) return 0; - const sourceLength = sourceBuf.length; - for(let i = 1; i < sourceLength; i++){ - if(sourceBuf[i] !== targetBuf[targetPos+i]) return 0; - } - return sourceLength; - } - __needMoreData(i, bufLen, end){ - if(end) return false; - const {quote} = this.options; - const {quoting, needMoreDataSize, recordDelimiterMaxLength} = this.state; - const numOfCharLeft = bufLen - i - 1; - const requiredLength = Math.max( - needMoreDataSize, - // Skip if the remaining buffer smaller than record delimiter - recordDelimiterMaxLength, - // Skip if the remaining buffer can be record delimiter following the closing quote - // 1 is for quote.length - quoting ? (quote.length + recordDelimiterMaxLength) : 0, - ); - return numOfCharLeft < requiredLength; - } - __isDelimiter(buf, pos, chr){ - const {delimiter, ignore_last_delimiters} = this.options; - if(ignore_last_delimiters === true && this.state.record.length === this.options.columns.length - 1){ - return 0; - }else if(ignore_last_delimiters !== false && typeof ignore_last_delimiters === 'number' && this.state.record.length === ignore_last_delimiters - 1){ - return 0; - } - loop1: for(let i = 0; i < delimiter.length; i++){ - const del = delimiter[i]; - if(del[0] === chr){ - for(let j = 1; j < del.length; j++){ - if(del[j] !== buf[pos+j]) continue loop1; + return [undefined, field]; + }, + // Helper to test if a character is a space or a line delimiter + __isCharTrimable: function(chr){ + return chr === space || chr === tab || chr === cr || chr === nl || chr === np; + }, + // Keep it in case we implement the `cast_int` option + // __isInt(value){ + // // return Number.isInteger(parseInt(value)) + // // return !isNaN( parseInt( obj ) ); + // return /^(\-|\+)?[1-9][0-9]*$/.test(value) + // } + __isFloat: function(value){ + return (value - parseFloat(value) + 1) >= 0; // Borrowed from jquery + }, + __compareBytes: function(sourceBuf, targetBuf, targetPos, firstByte){ + if(sourceBuf[0] !== firstByte) return 0; + const sourceLength = sourceBuf.length; + for(let i = 1; i < sourceLength; i++){ + if(sourceBuf[i] !== targetBuf[targetPos+i]) return 0; + } + return sourceLength; + }, + __isDelimiter: function(buf, pos, chr){ + const {delimiter, ignore_last_delimiters} = this.options; + if(ignore_last_delimiters === true && this.state.record.length === this.options.columns.length - 1){ + return 0; + }else if(ignore_last_delimiters !== false && typeof ignore_last_delimiters === 'number' && this.state.record.length === ignore_last_delimiters - 1){ + return 0; + } + loop1: for(let i = 0; i < delimiter.length; i++){ + const del = delimiter[i]; + if(del[0] === chr){ + for(let j = 1; j < del.length; j++){ + if(del[j] !== buf[pos+j]) continue loop1; + } + return del.length; } - return del.length; } - } - return 0; - } - __isRecordDelimiter(chr, buf, pos){ - const {record_delimiter} = this.options; - const recordDelimiterLength = record_delimiter.length; - loop1: for(let i = 0; i < recordDelimiterLength; i++){ - const rd = record_delimiter[i]; - const rdLength = rd.length; - if(rd[0] !== chr){ - continue; + return 0; + }, + __isRecordDelimiter: function(chr, buf, pos){ + const {record_delimiter} = this.options; + const recordDelimiterLength = record_delimiter.length; + loop1: for(let i = 0; i < recordDelimiterLength; i++){ + const rd = record_delimiter[i]; + const rdLength = rd.length; + if(rd[0] !== chr){ + continue; + } + for(let j = 1; j < rdLength; j++){ + if(rd[j] !== buf[pos+j]){ + continue loop1; + } + } + return rd.length; } - for(let j = 1; j < rdLength; j++){ - if(rd[j] !== buf[pos+j]){ - continue loop1; + return 0; + }, + __isEscape: function(buf, pos, chr){ + const {escape} = this.options; + if(escape === null) return false; + const l = escape.length; + if(escape[0] === chr){ + for(let i = 0; i < l; i++){ + if(escape[i] !== buf[pos+i]){ + return false; + } } + return true; } - return rd.length; - } - return 0; - } - __isEscape(buf, pos, chr){ - const {escape} = this.options; - if(escape === null) return false; - const l = escape.length; - if(escape[0] === chr){ + return false; + }, + __isQuote: function(buf, pos){ + const {quote} = this.options; + if(quote === null) return false; + const l = quote.length; for(let i = 0; i < l; i++){ - if(escape[i] !== buf[pos+i]){ + if(quote[i] !== buf[pos+i]){ return false; } } return true; - } - return false; - } - __isQuote(buf, pos){ - const {quote} = this.options; - if(quote === null) return false; - const l = quote.length; - for(let i = 0; i < l; i++){ - if(quote[i] !== buf[pos+i]){ - return false; - } - } - return true; - } - __autoDiscoverRecordDelimiter(buf, pos){ - const {encoding} = this.options; - const chr = buf[pos]; - if(chr === cr){ - if(buf[pos+1] === nl){ - this.options.record_delimiter.push(Buffer.from('\r\n', encoding)); - this.state.recordDelimiterMaxLength = 2; - return 2; - }else { - this.options.record_delimiter.push(Buffer.from('\r', encoding)); + }, + __autoDiscoverRecordDelimiter: function(buf, pos){ + const {encoding} = this.options; + const chr = buf[pos]; + if(chr === cr){ + if(buf[pos+1] === nl){ + this.options.record_delimiter.push(Buffer.from('\r\n', encoding)); + this.state.recordDelimiterMaxLength = 2; + return 2; + }else { + this.options.record_delimiter.push(Buffer.from('\r', encoding)); + this.state.recordDelimiterMaxLength = 1; + return 1; + } + }else if(chr === nl){ + this.options.record_delimiter.push(Buffer.from('\n', encoding)); this.state.recordDelimiterMaxLength = 1; return 1; } - }else if(chr === nl){ - this.options.record_delimiter.push(Buffer.from('\n', encoding)); - this.state.recordDelimiterMaxLength = 1; - return 1; - } - return 0; - } - __error(msg){ - const {encoding, raw, skip_records_with_error} = this.options; - const err = typeof msg === 'string' ? new Error(msg) : msg; - if(skip_records_with_error){ - this.state.recordHasError = true; - this.emit('skip', err, raw ? this.state.rawBuffer.toString(encoding) : undefined); - return undefined; - }else { - return err; + return 0; + }, + __error: function(msg){ + const {encoding, raw, skip_records_with_error} = this.options; + const err = typeof msg === 'string' ? new Error(msg) : msg; + if(skip_records_with_error){ + this.state.recordHasError = true; + if(this.options.on_skip !== undefined){ + this.options.on_skip(err, raw ? this.state.rawBuffer.toString(encoding) : undefined); + } + // this.emit('skip', err, raw ? this.state.rawBuffer.toString(encoding) : undefined); + return undefined; + }else { + return err; + } + }, + __infoDataSet: function(){ + return { + ...this.info, + columns: this.options.columns + }; + }, + __infoRecord: function(){ + const {columns, raw, encoding} = this.options; + return { + ...this.__infoDataSet(), + error: this.state.error, + header: columns === true, + index: this.state.record.length, + raw: raw ? this.state.rawBuffer.toString(encoding) : undefined + }; + }, + __infoField: function(){ + const {columns} = this.options; + const isColumns = Array.isArray(columns); + return { + ...this.__infoRecord(), + column: isColumns === true ? + (columns.length > this.state.record.length ? + columns[this.state.record.length].name : + null + ) : + this.state.record.length, + quoting: this.state.wasQuoting, + }; } - } - __infoDataSet(){ - return { - ...this.info, - columns: this.options.columns - }; - } - __infoRecord(){ - const {columns, raw, encoding} = this.options; - return { - ...this.__infoDataSet(), - error: this.state.error, - header: columns === true, - index: this.state.record.length, - raw: raw ? this.state.rawBuffer.toString(encoding) : undefined - }; - } - __infoField(){ - const {columns} = this.options; - const isColumns = Array.isArray(columns); - return { - ...this.__infoRecord(), - column: isColumns === true ? - (columns.length > this.state.record.length ? - columns[this.state.record.length].name : - null - ) : - this.state.record.length, - quoting: this.state.wasQuoting, - }; - } -} + }; +}; -const parse = function(data, options={}){ +const parse = function(data, opts={}){ if(typeof data === 'string'){ data = Buffer.from(data); } - const records = options && options.objname ? {} : []; - const parser = new Parser(options); - parser.push = function(record){ - if(record === null){ - return; - } - if(options.objname === undefined) + const records = opts && opts.objname ? {} : []; + const parser = transform(opts); + const push = (record) => { + if(parser.options.objname === undefined) records.push(record); else { records[record[0]] = record[1]; } }; - const err1 = parser.__parse(data, false); + const close = () => {}; + const err1 = parser.parse(data, false, push, close); if(err1 !== undefined) throw err1; - const err2 = parser.__parse(undefined, true); + const err2 = parser.parse(undefined, true, push, close); if(err2 !== undefined) throw err2; return records; }; diff --git a/packages/csv-parse/dist/esm/index.js b/packages/csv-parse/dist/esm/index.js index 4ae47e583..614a434ab 100644 --- a/packages/csv-parse/dist/esm/index.js +++ b/packages/csv-parse/dist/esm/index.js @@ -2627,7 +2627,7 @@ function format(f) { } }); for (var x = args[i]; i < len; x = args[++i]) { - if (isNull(x) || !isObject$1(x)) { + if (isNull(x) || !isObject(x)) { str += ' ' + x; } else { str += ' ' + inspect(x); @@ -3035,19 +3035,19 @@ function isUndefined(arg) { } function isRegExp(re) { - return isObject$1(re) && objectToString(re) === '[object RegExp]'; + return isObject(re) && objectToString(re) === '[object RegExp]'; } -function isObject$1(arg) { +function isObject(arg) { return typeof arg === 'object' && arg !== null; } function isDate(d) { - return isObject$1(d) && objectToString(d) === '[object Date]'; + return isObject(d) && objectToString(d) === '[object Date]'; } function isError(e) { - return isObject$1(e) && + return isObject(e) && (objectToString(e) === '[object Error]' || e instanceof Error); } @@ -3061,7 +3061,7 @@ function objectToString(o) { function _extend(origin, add) { // Don't do anything if add isn't an object - if (!add || !isObject$1(add)) return origin; + if (!add || !isObject(add)) return origin; var keys = Object.keys(add); var i = keys.length; @@ -4966,6 +4966,55 @@ Stream.prototype.pipe = function(dest, options) { return dest; }; +const is_object = function(obj){ + return (typeof obj === 'object' && obj !== null && !Array.isArray(obj)); +}; + +class CsvError extends Error { + constructor(code, message, options, ...contexts) { + if(Array.isArray(message)) message = message.join(' '); + super(message); + if(Error.captureStackTrace !== undefined){ + Error.captureStackTrace(this, CsvError); + } + this.code = code; + for(const context of contexts){ + for(const key in context){ + const value = context[key]; + this[key] = isBuffer(value) ? value.toString(options.encoding) : value == null ? value : JSON.parse(JSON.stringify(value)); + } + } + } +} + +const normalize_columns_array = function(columns){ + const normalizedColumns = []; + for(let i = 0, l = columns.length; i < l; i++){ + const column = columns[i]; + if(column === undefined || column === null || column === false){ + normalizedColumns[i] = { disabled: true }; + }else if(typeof column === 'string'){ + normalizedColumns[i] = { name: column }; + }else if(is_object(column)){ + if(typeof column.name !== 'string'){ + throw new CsvError('CSV_OPTION_COLUMNS_MISSING_NAME', [ + 'Option columns missing name:', + `property "name" is required at position ${i}`, + 'when column is an object literal' + ]); + } + normalizedColumns[i] = column; + }else { + throw new CsvError('CSV_INVALID_COLUMN_DEFINITION', [ + 'Invalid column definition:', + 'expect a string or a literal object,', + `got ${JSON.stringify(column)} at position ${i}` + ]); + } + } + return normalizedColumns; +}; + class ResizeableBuffer{ constructor(size=100){ this.size = size; @@ -5028,1210 +5077,1192 @@ class ResizeableBuffer{ } } -// white space characters -// https://en.wikipedia.org/wiki/Whitespace_character -// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions/Character_Classes#Types -// \f\n\r\t\v\u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff -const tab = 9; -const nl = 10; // \n, 0x0A in hexadecimal, 10 in decimal -const np = 12; -const cr = 13; // \r, 0x0D in hexadécimal, 13 in decimal -const space = 32; -const boms = { - // Note, the following are equals: - // Buffer.from("\ufeff") - // Buffer.from([239, 187, 191]) - // Buffer.from('EFBBBF', 'hex') - 'utf8': Buffer.from([239, 187, 191]), - // Note, the following are equals: - // Buffer.from "\ufeff", 'utf16le - // Buffer.from([255, 254]) - 'utf16le': Buffer.from([255, 254]) +const init_state = function(options){ + return { + bomSkipped: false, + bufBytesStart: 0, + castField: options.cast_function, + commenting: false, + // Current error encountered by a record + error: undefined, + enabled: options.from_line === 1, + escaping: false, + escapeIsQuote: isBuffer(options.escape) && isBuffer(options.quote) && Buffer.compare(options.escape, options.quote) === 0, + // columns can be `false`, `true`, `Array` + expectedRecordLength: Array.isArray(options.columns) ? options.columns.length : undefined, + field: new ResizeableBuffer(20), + firstLineToHeaders: options.cast_first_line_to_header, + needMoreDataSize: Math.max( + // Skip if the remaining buffer smaller than comment + options.comment !== null ? options.comment.length : 0, + // Skip if the remaining buffer can be delimiter + ...options.delimiter.map((delimiter) => delimiter.length), + // Skip if the remaining buffer can be escape sequence + options.quote !== null ? options.quote.length : 0, + ), + previousBuf: undefined, + quoting: false, + stop: false, + rawBuffer: new ResizeableBuffer(100), + record: [], + recordHasError: false, + record_length: 0, + recordDelimiterMaxLength: options.record_delimiter.length === 0 ? 2 : Math.max(...options.record_delimiter.map((v) => v.length)), + trimChars: [Buffer.from(' ', options.encoding)[0], Buffer.from('\t', options.encoding)[0]], + wasQuoting: false, + wasRowDelimiter: false + }; }; -class CsvError extends Error { - constructor(code, message, options, ...contexts) { - if(Array.isArray(message)) message = message.join(' '); - super(message); - if(Error.captureStackTrace !== undefined){ - Error.captureStackTrace(this, CsvError); - } - this.code = code; - for(const context of contexts){ - for(const key in context){ - const value = context[key]; - this[key] = isBuffer(value) ? value.toString(options.encoding) : value == null ? value : JSON.parse(JSON.stringify(value)); - } - } - } -} - const underscore = function(str){ return str.replace(/([A-Z])/g, function(_, match){ return '_' + match.toLowerCase(); }); }; -const isObject = function(obj){ - return (typeof obj === 'object' && obj !== null && !Array.isArray(obj)); -}; - -const isRecordEmpty = function(record){ - return record.every((field) => field == null || field.toString && field.toString().trim() === ''); -}; - -const normalizeColumnsArray = function(columns){ - const normalizedColumns = []; - for(let i = 0, l = columns.length; i < l; i++){ - const column = columns[i]; - if(column === undefined || column === null || column === false){ - normalizedColumns[i] = { disabled: true }; - }else if(typeof column === 'string'){ - normalizedColumns[i] = { name: column }; - }else if(isObject(column)){ - if(typeof column.name !== 'string'){ - throw new CsvError('CSV_OPTION_COLUMNS_MISSING_NAME', [ - 'Option columns missing name:', - `property "name" is required at position ${i}`, - 'when column is an object literal' - ]); - } - normalizedColumns[i] = column; - }else { - throw new CsvError('CSV_INVALID_COLUMN_DEFINITION', [ - 'Invalid column definition:', - 'expect a string or a literal object,', - `got ${JSON.stringify(column)} at position ${i}` - ]); - } - } - return normalizedColumns; -}; - -class Parser extends Transform { - constructor(opts = {}){ - super({...{readableObjectMode: true}, ...opts, encoding: null}); - this.__originalOptions = opts; - this.__normalizeOptions(opts); - } - __normalizeOptions(opts){ - const options = {}; - // Merge with user options - for(const opt in opts){ - options[underscore(opt)] = opts[opt]; - } - // Normalize option `encoding` - // Note: defined first because other options depends on it - // to convert chars/strings into buffers. - if(options.encoding === undefined || options.encoding === true){ - options.encoding = 'utf8'; - }else if(options.encoding === null || options.encoding === false){ - options.encoding = null; - }else if(typeof options.encoding !== 'string' && options.encoding !== null){ - throw new CsvError('CSV_INVALID_OPTION_ENCODING', [ - 'Invalid option encoding:', - 'encoding must be a string or null to return a buffer,', - `got ${JSON.stringify(options.encoding)}` - ], options); - } - // Normalize option `bom` - if(options.bom === undefined || options.bom === null || options.bom === false){ - options.bom = false; - }else if(options.bom !== true){ - throw new CsvError('CSV_INVALID_OPTION_BOM', [ - 'Invalid option bom:', 'bom must be true,', - `got ${JSON.stringify(options.bom)}` - ], options); - } - // Normalize option `cast` - let fnCastField = null; - if(options.cast === undefined || options.cast === null || options.cast === false || options.cast === ''){ - options.cast = undefined; - }else if(typeof options.cast === 'function'){ - fnCastField = options.cast; - options.cast = true; - }else if(options.cast !== true){ - throw new CsvError('CSV_INVALID_OPTION_CAST', [ - 'Invalid option cast:', 'cast must be true or a function,', - `got ${JSON.stringify(options.cast)}` - ], options); - } - // Normalize option `cast_date` - if(options.cast_date === undefined || options.cast_date === null || options.cast_date === false || options.cast_date === ''){ - options.cast_date = false; - }else if(options.cast_date === true){ - options.cast_date = function(value){ - const date = Date.parse(value); - return !isNaN(date) ? new Date(date) : value; - }; - }else { - throw new CsvError('CSV_INVALID_OPTION_CAST_DATE', [ - 'Invalid option cast_date:', 'cast_date must be true or a function,', - `got ${JSON.stringify(options.cast_date)}` +const normalize_options = function(opts){ + const options = {}; + // Merge with user options + for(const opt in opts){ + options[underscore(opt)] = opts[opt]; + } + // Normalize option `encoding` + // Note: defined first because other options depends on it + // to convert chars/strings into buffers. + if(options.encoding === undefined || options.encoding === true){ + options.encoding = 'utf8'; + }else if(options.encoding === null || options.encoding === false){ + options.encoding = null; + }else if(typeof options.encoding !== 'string' && options.encoding !== null){ + throw new CsvError('CSV_INVALID_OPTION_ENCODING', [ + 'Invalid option encoding:', + 'encoding must be a string or null to return a buffer,', + `got ${JSON.stringify(options.encoding)}` + ], options); + } + // Normalize option `bom` + if(options.bom === undefined || options.bom === null || options.bom === false){ + options.bom = false; + }else if(options.bom !== true){ + throw new CsvError('CSV_INVALID_OPTION_BOM', [ + 'Invalid option bom:', 'bom must be true,', + `got ${JSON.stringify(options.bom)}` + ], options); + } + // Normalize option `cast` + options.cast_function = null; + if(options.cast === undefined || options.cast === null || options.cast === false || options.cast === ''){ + options.cast = undefined; + }else if(typeof options.cast === 'function'){ + options.cast_function = options.cast; + options.cast = true; + }else if(options.cast !== true){ + throw new CsvError('CSV_INVALID_OPTION_CAST', [ + 'Invalid option cast:', 'cast must be true or a function,', + `got ${JSON.stringify(options.cast)}` + ], options); + } + // Normalize option `cast_date` + if(options.cast_date === undefined || options.cast_date === null || options.cast_date === false || options.cast_date === ''){ + options.cast_date = false; + }else if(options.cast_date === true){ + options.cast_date = function(value){ + const date = Date.parse(value); + return !isNaN(date) ? new Date(date) : value; + }; + }else { + throw new CsvError('CSV_INVALID_OPTION_CAST_DATE', [ + 'Invalid option cast_date:', 'cast_date must be true or a function,', + `got ${JSON.stringify(options.cast_date)}` + ], options); + } + // Normalize option `columns` + options.cast_first_line_to_header = null; + if(options.columns === true){ + // Fields in the first line are converted as-is to columns + options.cast_first_line_to_header = undefined; + }else if(typeof options.columns === 'function'){ + options.cast_first_line_to_header = options.columns; + options.columns = true; + }else if(Array.isArray(options.columns)){ + options.columns = normalize_columns_array(options.columns); + }else if(options.columns === undefined || options.columns === null || options.columns === false){ + options.columns = false; + }else { + throw new CsvError('CSV_INVALID_OPTION_COLUMNS', [ + 'Invalid option columns:', + 'expect an array, a function or true,', + `got ${JSON.stringify(options.columns)}` + ], options); + } + // Normalize option `group_columns_by_name` + if(options.group_columns_by_name === undefined || options.group_columns_by_name === null || options.group_columns_by_name === false){ + options.group_columns_by_name = false; + }else if(options.group_columns_by_name !== true){ + throw new CsvError('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ + 'Invalid option group_columns_by_name:', + 'expect an boolean,', + `got ${JSON.stringify(options.group_columns_by_name)}` + ], options); + }else if(options.columns === false){ + throw new CsvError('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ + 'Invalid option group_columns_by_name:', + 'the `columns` mode must be activated.' + ], options); + } + // Normalize option `comment` + if(options.comment === undefined || options.comment === null || options.comment === false || options.comment === ''){ + options.comment = null; + }else { + if(typeof options.comment === 'string'){ + options.comment = Buffer.from(options.comment, options.encoding); + } + if(!isBuffer(options.comment)){ + throw new CsvError('CSV_INVALID_OPTION_COMMENT', [ + 'Invalid option comment:', + 'comment must be a buffer or a string,', + `got ${JSON.stringify(options.comment)}` ], options); } - // Normalize option `columns` - let fnFirstLineToHeaders = null; - if(options.columns === true){ - // Fields in the first line are converted as-is to columns - fnFirstLineToHeaders = undefined; - }else if(typeof options.columns === 'function'){ - fnFirstLineToHeaders = options.columns; - options.columns = true; - }else if(Array.isArray(options.columns)){ - options.columns = normalizeColumnsArray(options.columns); - }else if(options.columns === undefined || options.columns === null || options.columns === false){ - options.columns = false; - }else { - throw new CsvError('CSV_INVALID_OPTION_COLUMNS', [ - 'Invalid option columns:', - 'expect an array, a function or true,', - `got ${JSON.stringify(options.columns)}` - ], options); + } + // Normalize option `delimiter` + const delimiter_json = JSON.stringify(options.delimiter); + if(!Array.isArray(options.delimiter)) options.delimiter = [options.delimiter]; + if(options.delimiter.length === 0){ + throw new CsvError('CSV_INVALID_OPTION_DELIMITER', [ + 'Invalid option delimiter:', + 'delimiter must be a non empty string or buffer or array of string|buffer,', + `got ${delimiter_json}` + ], options); + } + options.delimiter = options.delimiter.map(function(delimiter){ + if(delimiter === undefined || delimiter === null || delimiter === false){ + return Buffer.from(',', options.encoding); } - // Normalize option `group_columns_by_name` - if(options.group_columns_by_name === undefined || options.group_columns_by_name === null || options.group_columns_by_name === false){ - options.group_columns_by_name = false; - }else if(options.group_columns_by_name !== true){ - throw new CsvError('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ - 'Invalid option group_columns_by_name:', - 'expect an boolean,', - `got ${JSON.stringify(options.group_columns_by_name)}` - ], options); - }else if(options.columns === false){ - throw new CsvError('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ - 'Invalid option group_columns_by_name:', - 'the `columns` mode must be activated.' - ], options); + if(typeof delimiter === 'string'){ + delimiter = Buffer.from(delimiter, options.encoding); } - // Normalize option `comment` - if(options.comment === undefined || options.comment === null || options.comment === false || options.comment === ''){ - options.comment = null; - }else { - if(typeof options.comment === 'string'){ - options.comment = Buffer.from(options.comment, options.encoding); - } - if(!isBuffer(options.comment)){ - throw new CsvError('CSV_INVALID_OPTION_COMMENT', [ - 'Invalid option comment:', - 'comment must be a buffer or a string,', - `got ${JSON.stringify(options.comment)}` - ], options); - } - } - // Normalize option `delimiter` - const delimiter_json = JSON.stringify(options.delimiter); - if(!Array.isArray(options.delimiter)) options.delimiter = [options.delimiter]; - if(options.delimiter.length === 0){ + if(!isBuffer(delimiter) || delimiter.length === 0){ throw new CsvError('CSV_INVALID_OPTION_DELIMITER', [ 'Invalid option delimiter:', 'delimiter must be a non empty string or buffer or array of string|buffer,', `got ${delimiter_json}` ], options); } - options.delimiter = options.delimiter.map(function(delimiter){ - if(delimiter === undefined || delimiter === null || delimiter === false){ - return Buffer.from(',', options.encoding); - } - if(typeof delimiter === 'string'){ - delimiter = Buffer.from(delimiter, options.encoding); - } - if(!isBuffer(delimiter) || delimiter.length === 0){ - throw new CsvError('CSV_INVALID_OPTION_DELIMITER', [ - 'Invalid option delimiter:', - 'delimiter must be a non empty string or buffer or array of string|buffer,', - `got ${delimiter_json}` - ], options); - } - return delimiter; - }); - // Normalize option `escape` - if(options.escape === undefined || options.escape === true){ - options.escape = Buffer.from('"', options.encoding); - }else if(typeof options.escape === 'string'){ - options.escape = Buffer.from(options.escape, options.encoding); - }else if (options.escape === null || options.escape === false){ - options.escape = null; - } - if(options.escape !== null){ - if(!isBuffer(options.escape)){ - throw new Error(`Invalid Option: escape must be a buffer, a string or a boolean, got ${JSON.stringify(options.escape)}`); + return delimiter; + }); + // Normalize option `escape` + if(options.escape === undefined || options.escape === true){ + options.escape = Buffer.from('"', options.encoding); + }else if(typeof options.escape === 'string'){ + options.escape = Buffer.from(options.escape, options.encoding); + }else if (options.escape === null || options.escape === false){ + options.escape = null; + } + if(options.escape !== null){ + if(!isBuffer(options.escape)){ + throw new Error(`Invalid Option: escape must be a buffer, a string or a boolean, got ${JSON.stringify(options.escape)}`); + } + } + // Normalize option `from` + if(options.from === undefined || options.from === null){ + options.from = 1; + }else { + if(typeof options.from === 'string' && /\d+/.test(options.from)){ + options.from = parseInt(options.from); + } + if(Number.isInteger(options.from)){ + if(options.from < 0){ + throw new Error(`Invalid Option: from must be a positive integer, got ${JSON.stringify(opts.from)}`); } - } - // Normalize option `from` - if(options.from === undefined || options.from === null){ - options.from = 1; }else { - if(typeof options.from === 'string' && /\d+/.test(options.from)){ - options.from = parseInt(options.from); - } - if(Number.isInteger(options.from)){ - if(options.from < 0){ - throw new Error(`Invalid Option: from must be a positive integer, got ${JSON.stringify(opts.from)}`); - } - }else { - throw new Error(`Invalid Option: from must be an integer, got ${JSON.stringify(options.from)}`); - } + throw new Error(`Invalid Option: from must be an integer, got ${JSON.stringify(options.from)}`); } - // Normalize option `from_line` - if(options.from_line === undefined || options.from_line === null){ - options.from_line = 1; - }else { - if(typeof options.from_line === 'string' && /\d+/.test(options.from_line)){ - options.from_line = parseInt(options.from_line); - } - if(Number.isInteger(options.from_line)){ - if(options.from_line <= 0){ - throw new Error(`Invalid Option: from_line must be a positive integer greater than 0, got ${JSON.stringify(opts.from_line)}`); - } - }else { - throw new Error(`Invalid Option: from_line must be an integer, got ${JSON.stringify(opts.from_line)}`); - } + } + // Normalize option `from_line` + if(options.from_line === undefined || options.from_line === null){ + options.from_line = 1; + }else { + if(typeof options.from_line === 'string' && /\d+/.test(options.from_line)){ + options.from_line = parseInt(options.from_line); } - // Normalize options `ignore_last_delimiters` - if(options.ignore_last_delimiters === undefined || options.ignore_last_delimiters === null){ - options.ignore_last_delimiters = false; - }else if(typeof options.ignore_last_delimiters === 'number'){ - options.ignore_last_delimiters = Math.floor(options.ignore_last_delimiters); - if(options.ignore_last_delimiters === 0){ - options.ignore_last_delimiters = false; + if(Number.isInteger(options.from_line)){ + if(options.from_line <= 0){ + throw new Error(`Invalid Option: from_line must be a positive integer greater than 0, got ${JSON.stringify(opts.from_line)}`); } - }else if(typeof options.ignore_last_delimiters !== 'boolean'){ - throw new CsvError('CSV_INVALID_OPTION_IGNORE_LAST_DELIMITERS', [ - 'Invalid option `ignore_last_delimiters`:', - 'the value must be a boolean value or an integer,', - `got ${JSON.stringify(options.ignore_last_delimiters)}` - ], options); - } - if(options.ignore_last_delimiters === true && options.columns === false){ - throw new CsvError('CSV_IGNORE_LAST_DELIMITERS_REQUIRES_COLUMNS', [ - 'The option `ignore_last_delimiters`', - 'requires the activation of the `columns` option' - ], options); - } - // Normalize option `info` - if(options.info === undefined || options.info === null || options.info === false){ - options.info = false; - }else if(options.info !== true){ - throw new Error(`Invalid Option: info must be true, got ${JSON.stringify(options.info)}`); - } - // Normalize option `max_record_size` - if(options.max_record_size === undefined || options.max_record_size === null || options.max_record_size === false){ - options.max_record_size = 0; - }else if(Number.isInteger(options.max_record_size) && options.max_record_size >= 0);else if(typeof options.max_record_size === 'string' && /\d+/.test(options.max_record_size)){ - options.max_record_size = parseInt(options.max_record_size); }else { - throw new Error(`Invalid Option: max_record_size must be a positive integer, got ${JSON.stringify(options.max_record_size)}`); - } - // Normalize option `objname` - if(options.objname === undefined || options.objname === null || options.objname === false){ - options.objname = undefined; - }else if(isBuffer(options.objname)){ - if(options.objname.length === 0){ - throw new Error(`Invalid Option: objname must be a non empty buffer`); - } - if(options.encoding === null);else { - options.objname = options.objname.toString(options.encoding); - } - }else if(typeof options.objname === 'string'){ - if(options.objname.length === 0){ - throw new Error(`Invalid Option: objname must be a non empty string`); - } - // Great, nothing to do - }else if(typeof options.objname === 'number');else { - throw new Error(`Invalid Option: objname must be a string or a buffer, got ${options.objname}`); - } - if(options.objname !== undefined){ - if(typeof options.objname === 'number'){ - if(options.columns !== false){ - throw Error('Invalid Option: objname index cannot be combined with columns or be defined as a field'); - } - }else { // A string or a buffer - if(options.columns === false){ - throw Error('Invalid Option: objname field must be combined with columns or be defined as an index'); - } - } + throw new Error(`Invalid Option: from_line must be an integer, got ${JSON.stringify(opts.from_line)}`); } - // Normalize option `on_record` - if(options.on_record === undefined || options.on_record === null){ - options.on_record = undefined; - }else if(typeof options.on_record !== 'function'){ - throw new CsvError('CSV_INVALID_OPTION_ON_RECORD', [ - 'Invalid option `on_record`:', - 'expect a function,', - `got ${JSON.stringify(options.on_record)}` - ], options); + } + // Normalize options `ignore_last_delimiters` + if(options.ignore_last_delimiters === undefined || options.ignore_last_delimiters === null){ + options.ignore_last_delimiters = false; + }else if(typeof options.ignore_last_delimiters === 'number'){ + options.ignore_last_delimiters = Math.floor(options.ignore_last_delimiters); + if(options.ignore_last_delimiters === 0){ + options.ignore_last_delimiters = false; } - // Normalize option `quote` - if(options.quote === null || options.quote === false || options.quote === ''){ - options.quote = null; - }else { - if(options.quote === undefined || options.quote === true){ - options.quote = Buffer.from('"', options.encoding); - }else if(typeof options.quote === 'string'){ - options.quote = Buffer.from(options.quote, options.encoding); + }else if(typeof options.ignore_last_delimiters !== 'boolean'){ + throw new CsvError('CSV_INVALID_OPTION_IGNORE_LAST_DELIMITERS', [ + 'Invalid option `ignore_last_delimiters`:', + 'the value must be a boolean value or an integer,', + `got ${JSON.stringify(options.ignore_last_delimiters)}` + ], options); + } + if(options.ignore_last_delimiters === true && options.columns === false){ + throw new CsvError('CSV_IGNORE_LAST_DELIMITERS_REQUIRES_COLUMNS', [ + 'The option `ignore_last_delimiters`', + 'requires the activation of the `columns` option' + ], options); + } + // Normalize option `info` + if(options.info === undefined || options.info === null || options.info === false){ + options.info = false; + }else if(options.info !== true){ + throw new Error(`Invalid Option: info must be true, got ${JSON.stringify(options.info)}`); + } + // Normalize option `max_record_size` + if(options.max_record_size === undefined || options.max_record_size === null || options.max_record_size === false){ + options.max_record_size = 0; + }else if(Number.isInteger(options.max_record_size) && options.max_record_size >= 0);else if(typeof options.max_record_size === 'string' && /\d+/.test(options.max_record_size)){ + options.max_record_size = parseInt(options.max_record_size); + }else { + throw new Error(`Invalid Option: max_record_size must be a positive integer, got ${JSON.stringify(options.max_record_size)}`); + } + // Normalize option `objname` + if(options.objname === undefined || options.objname === null || options.objname === false){ + options.objname = undefined; + }else if(isBuffer(options.objname)){ + if(options.objname.length === 0){ + throw new Error(`Invalid Option: objname must be a non empty buffer`); + } + if(options.encoding === null);else { + options.objname = options.objname.toString(options.encoding); + } + }else if(typeof options.objname === 'string'){ + if(options.objname.length === 0){ + throw new Error(`Invalid Option: objname must be a non empty string`); + } + // Great, nothing to do + }else if(typeof options.objname === 'number');else { + throw new Error(`Invalid Option: objname must be a string or a buffer, got ${options.objname}`); + } + if(options.objname !== undefined){ + if(typeof options.objname === 'number'){ + if(options.columns !== false){ + throw Error('Invalid Option: objname index cannot be combined with columns or be defined as a field'); } - if(!isBuffer(options.quote)){ - throw new Error(`Invalid Option: quote must be a buffer or a string, got ${JSON.stringify(options.quote)}`); + }else { // A string or a buffer + if(options.columns === false){ + throw Error('Invalid Option: objname field must be combined with columns or be defined as an index'); } } - // Normalize option `raw` - if(options.raw === undefined || options.raw === null || options.raw === false){ - options.raw = false; - }else if(options.raw !== true){ - throw new Error(`Invalid Option: raw must be true, got ${JSON.stringify(options.raw)}`); - } - // Normalize option `record_delimiter` - if(options.record_delimiter === undefined){ - options.record_delimiter = []; - }else if(typeof options.record_delimiter === 'string' || isBuffer(options.record_delimiter)){ - if(options.record_delimiter.length === 0){ - throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ - 'Invalid option `record_delimiter`:', - 'value must be a non empty string or buffer,', - `got ${JSON.stringify(options.record_delimiter)}` - ], options); - } - options.record_delimiter = [options.record_delimiter]; - }else if(!Array.isArray(options.record_delimiter)){ + } + // Normalize option `on_record` + if(options.on_record === undefined || options.on_record === null){ + options.on_record = undefined; + }else if(typeof options.on_record !== 'function'){ + throw new CsvError('CSV_INVALID_OPTION_ON_RECORD', [ + 'Invalid option `on_record`:', + 'expect a function,', + `got ${JSON.stringify(options.on_record)}` + ], options); + } + // Normalize option `quote` + if(options.quote === null || options.quote === false || options.quote === ''){ + options.quote = null; + }else { + if(options.quote === undefined || options.quote === true){ + options.quote = Buffer.from('"', options.encoding); + }else if(typeof options.quote === 'string'){ + options.quote = Buffer.from(options.quote, options.encoding); + } + if(!isBuffer(options.quote)){ + throw new Error(`Invalid Option: quote must be a buffer or a string, got ${JSON.stringify(options.quote)}`); + } + } + // Normalize option `raw` + if(options.raw === undefined || options.raw === null || options.raw === false){ + options.raw = false; + }else if(options.raw !== true){ + throw new Error(`Invalid Option: raw must be true, got ${JSON.stringify(options.raw)}`); + } + // Normalize option `record_delimiter` + if(options.record_delimiter === undefined){ + options.record_delimiter = []; + }else if(typeof options.record_delimiter === 'string' || isBuffer(options.record_delimiter)){ + if(options.record_delimiter.length === 0){ throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ 'Invalid option `record_delimiter`:', - 'value must be a string, a buffer or array of string|buffer,', + 'value must be a non empty string or buffer,', `got ${JSON.stringify(options.record_delimiter)}` ], options); } - options.record_delimiter = options.record_delimiter.map(function(rd, i){ - if(typeof rd !== 'string' && ! isBuffer(rd)){ - throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ - 'Invalid option `record_delimiter`:', - 'value must be a string, a buffer or array of string|buffer', - `at index ${i},`, - `got ${JSON.stringify(rd)}` - ], options); - }else if(rd.length === 0){ - throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ - 'Invalid option `record_delimiter`:', - 'value must be a non empty string or buffer', - `at index ${i},`, - `got ${JSON.stringify(rd)}` - ], options); - } - if(typeof rd === 'string'){ - rd = Buffer.from(rd, options.encoding); - } - return rd; - }); - // Normalize option `relax_column_count` - if(typeof options.relax_column_count === 'boolean');else if(options.relax_column_count === undefined || options.relax_column_count === null){ - options.relax_column_count = false; - }else { - throw new Error(`Invalid Option: relax_column_count must be a boolean, got ${JSON.stringify(options.relax_column_count)}`); - } - if(typeof options.relax_column_count_less === 'boolean');else if(options.relax_column_count_less === undefined || options.relax_column_count_less === null){ - options.relax_column_count_less = false; - }else { - throw new Error(`Invalid Option: relax_column_count_less must be a boolean, got ${JSON.stringify(options.relax_column_count_less)}`); + options.record_delimiter = [options.record_delimiter]; + }else if(!Array.isArray(options.record_delimiter)){ + throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ + 'Invalid option `record_delimiter`:', + 'value must be a string, a buffer or array of string|buffer,', + `got ${JSON.stringify(options.record_delimiter)}` + ], options); + } + options.record_delimiter = options.record_delimiter.map(function(rd, i){ + if(typeof rd !== 'string' && ! isBuffer(rd)){ + throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ + 'Invalid option `record_delimiter`:', + 'value must be a string, a buffer or array of string|buffer', + `at index ${i},`, + `got ${JSON.stringify(rd)}` + ], options); + }else if(rd.length === 0){ + throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ + 'Invalid option `record_delimiter`:', + 'value must be a non empty string or buffer', + `at index ${i},`, + `got ${JSON.stringify(rd)}` + ], options); } - if(typeof options.relax_column_count_more === 'boolean');else if(options.relax_column_count_more === undefined || options.relax_column_count_more === null){ - options.relax_column_count_more = false; - }else { - throw new Error(`Invalid Option: relax_column_count_more must be a boolean, got ${JSON.stringify(options.relax_column_count_more)}`); + if(typeof rd === 'string'){ + rd = Buffer.from(rd, options.encoding); } - // Normalize option `relax_quotes` - if(typeof options.relax_quotes === 'boolean');else if(options.relax_quotes === undefined || options.relax_quotes === null){ - options.relax_quotes = false; + return rd; + }); + // Normalize option `relax_column_count` + if(typeof options.relax_column_count === 'boolean');else if(options.relax_column_count === undefined || options.relax_column_count === null){ + options.relax_column_count = false; + }else { + throw new Error(`Invalid Option: relax_column_count must be a boolean, got ${JSON.stringify(options.relax_column_count)}`); + } + if(typeof options.relax_column_count_less === 'boolean');else if(options.relax_column_count_less === undefined || options.relax_column_count_less === null){ + options.relax_column_count_less = false; + }else { + throw new Error(`Invalid Option: relax_column_count_less must be a boolean, got ${JSON.stringify(options.relax_column_count_less)}`); + } + if(typeof options.relax_column_count_more === 'boolean');else if(options.relax_column_count_more === undefined || options.relax_column_count_more === null){ + options.relax_column_count_more = false; + }else { + throw new Error(`Invalid Option: relax_column_count_more must be a boolean, got ${JSON.stringify(options.relax_column_count_more)}`); + } + // Normalize option `relax_quotes` + if(typeof options.relax_quotes === 'boolean');else if(options.relax_quotes === undefined || options.relax_quotes === null){ + options.relax_quotes = false; + }else { + throw new Error(`Invalid Option: relax_quotes must be a boolean, got ${JSON.stringify(options.relax_quotes)}`); + } + // Normalize option `skip_empty_lines` + if(typeof options.skip_empty_lines === 'boolean');else if(options.skip_empty_lines === undefined || options.skip_empty_lines === null){ + options.skip_empty_lines = false; + }else { + throw new Error(`Invalid Option: skip_empty_lines must be a boolean, got ${JSON.stringify(options.skip_empty_lines)}`); + } + // Normalize option `skip_records_with_empty_values` + if(typeof options.skip_records_with_empty_values === 'boolean');else if(options.skip_records_with_empty_values === undefined || options.skip_records_with_empty_values === null){ + options.skip_records_with_empty_values = false; + }else { + throw new Error(`Invalid Option: skip_records_with_empty_values must be a boolean, got ${JSON.stringify(options.skip_records_with_empty_values)}`); + } + // Normalize option `skip_records_with_error` + if(typeof options.skip_records_with_error === 'boolean');else if(options.skip_records_with_error === undefined || options.skip_records_with_error === null){ + options.skip_records_with_error = false; + }else { + throw new Error(`Invalid Option: skip_records_with_error must be a boolean, got ${JSON.stringify(options.skip_records_with_error)}`); + } + // Normalize option `rtrim` + if(options.rtrim === undefined || options.rtrim === null || options.rtrim === false){ + options.rtrim = false; + }else if(options.rtrim !== true){ + throw new Error(`Invalid Option: rtrim must be a boolean, got ${JSON.stringify(options.rtrim)}`); + } + // Normalize option `ltrim` + if(options.ltrim === undefined || options.ltrim === null || options.ltrim === false){ + options.ltrim = false; + }else if(options.ltrim !== true){ + throw new Error(`Invalid Option: ltrim must be a boolean, got ${JSON.stringify(options.ltrim)}`); + } + // Normalize option `trim` + if(options.trim === undefined || options.trim === null || options.trim === false){ + options.trim = false; + }else if(options.trim !== true){ + throw new Error(`Invalid Option: trim must be a boolean, got ${JSON.stringify(options.trim)}`); + } + // Normalize options `trim`, `ltrim` and `rtrim` + if(options.trim === true && opts.ltrim !== false){ + options.ltrim = true; + }else if(options.ltrim !== true){ + options.ltrim = false; + } + if(options.trim === true && opts.rtrim !== false){ + options.rtrim = true; + }else if(options.rtrim !== true){ + options.rtrim = false; + } + // Normalize option `to` + if(options.to === undefined || options.to === null){ + options.to = -1; + }else { + if(typeof options.to === 'string' && /\d+/.test(options.to)){ + options.to = parseInt(options.to); + } + if(Number.isInteger(options.to)){ + if(options.to <= 0){ + throw new Error(`Invalid Option: to must be a positive integer greater than 0, got ${JSON.stringify(opts.to)}`); + } }else { - throw new Error(`Invalid Option: relax_quotes must be a boolean, got ${JSON.stringify(options.relax_quotes)}`); + throw new Error(`Invalid Option: to must be an integer, got ${JSON.stringify(opts.to)}`); } - // Normalize option `skip_empty_lines` - if(typeof options.skip_empty_lines === 'boolean');else if(options.skip_empty_lines === undefined || options.skip_empty_lines === null){ - options.skip_empty_lines = false; - }else { - throw new Error(`Invalid Option: skip_empty_lines must be a boolean, got ${JSON.stringify(options.skip_empty_lines)}`); + } + // Normalize option `to_line` + if(options.to_line === undefined || options.to_line === null){ + options.to_line = -1; + }else { + if(typeof options.to_line === 'string' && /\d+/.test(options.to_line)){ + options.to_line = parseInt(options.to_line); } - // Normalize option `skip_records_with_empty_values` - if(typeof options.skip_records_with_empty_values === 'boolean');else if(options.skip_records_with_empty_values === undefined || options.skip_records_with_empty_values === null){ - options.skip_records_with_empty_values = false; + if(Number.isInteger(options.to_line)){ + if(options.to_line <= 0){ + throw new Error(`Invalid Option: to_line must be a positive integer greater than 0, got ${JSON.stringify(opts.to_line)}`); + } }else { - throw new Error(`Invalid Option: skip_records_with_empty_values must be a boolean, got ${JSON.stringify(options.skip_records_with_empty_values)}`); + throw new Error(`Invalid Option: to_line must be an integer, got ${JSON.stringify(opts.to_line)}`); } - // Normalize option `skip_records_with_error` - if(typeof options.skip_records_with_error === 'boolean');else if(options.skip_records_with_error === undefined || options.skip_records_with_error === null){ - options.skip_records_with_error = false; - }else { - throw new Error(`Invalid Option: skip_records_with_error must be a boolean, got ${JSON.stringify(options.skip_records_with_error)}`); - } - // Normalize option `rtrim` - if(options.rtrim === undefined || options.rtrim === null || options.rtrim === false){ - options.rtrim = false; - }else if(options.rtrim !== true){ - throw new Error(`Invalid Option: rtrim must be a boolean, got ${JSON.stringify(options.rtrim)}`); - } - // Normalize option `ltrim` - if(options.ltrim === undefined || options.ltrim === null || options.ltrim === false){ - options.ltrim = false; - }else if(options.ltrim !== true){ - throw new Error(`Invalid Option: ltrim must be a boolean, got ${JSON.stringify(options.ltrim)}`); - } - // Normalize option `trim` - if(options.trim === undefined || options.trim === null || options.trim === false){ - options.trim = false; - }else if(options.trim !== true){ - throw new Error(`Invalid Option: trim must be a boolean, got ${JSON.stringify(options.trim)}`); - } - // Normalize options `trim`, `ltrim` and `rtrim` - if(options.trim === true && opts.ltrim !== false){ - options.ltrim = true; - }else if(options.ltrim !== true){ - options.ltrim = false; - } - if(options.trim === true && opts.rtrim !== false){ - options.rtrim = true; - }else if(options.rtrim !== true){ - options.rtrim = false; - } - // Normalize option `to` - if(options.to === undefined || options.to === null){ - options.to = -1; - }else { - if(typeof options.to === 'string' && /\d+/.test(options.to)){ - options.to = parseInt(options.to); - } - if(Number.isInteger(options.to)){ - if(options.to <= 0){ - throw new Error(`Invalid Option: to must be a positive integer greater than 0, got ${JSON.stringify(opts.to)}`); + } + return options; +}; + +const isRecordEmpty = function(record){ + return record.every((field) => field == null || field.toString && field.toString().trim() === ''); +}; + +// white space characters +// https://en.wikipedia.org/wiki/Whitespace_character +// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions/Character_Classes#Types +// \f\n\r\t\v\u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff +const tab = 9; +const nl = 10; // \n, 0x0A in hexadecimal, 10 in decimal +const np = 12; +const cr = 13; // \r, 0x0D in hexadécimal, 13 in decimal +const space = 32; +const boms = { + // Note, the following are equals: + // Buffer.from("\ufeff") + // Buffer.from([239, 187, 191]) + // Buffer.from('EFBBBF', 'hex') + 'utf8': Buffer.from([239, 187, 191]), + // Note, the following are equals: + // Buffer.from "\ufeff", 'utf16le + // Buffer.from([255, 254]) + 'utf16le': Buffer.from([255, 254]) +}; + +const transform = function(original_options = {}) { + const info = { + bytes: 0, + comment_lines: 0, + empty_lines: 0, + invalid_field_length: 0, + lines: 1, + records: 0 + }; + const options = normalize_options(original_options); + return { + info: info, + original_options: original_options, + options: options, + state: init_state(options), + __needMoreData: function(i, bufLen, end){ + if(end) return false; + const {quote} = this.options; + const {quoting, needMoreDataSize, recordDelimiterMaxLength} = this.state; + const numOfCharLeft = bufLen - i - 1; + const requiredLength = Math.max( + needMoreDataSize, + // Skip if the remaining buffer smaller than record delimiter + recordDelimiterMaxLength, + // Skip if the remaining buffer can be record delimiter following the closing quote + // 1 is for quote.length + quoting ? (quote.length + recordDelimiterMaxLength) : 0, + ); + return numOfCharLeft < requiredLength; + }, + // Central parser implementation + parse: function(nextBuf, end, push, close){ + const {bom, comment, escape, from_line, ltrim, max_record_size, quote, raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options; + let {record_delimiter} = this.options; + const {bomSkipped, previousBuf, rawBuffer, escapeIsQuote} = this.state; + let buf; + if(previousBuf === undefined){ + if(nextBuf === undefined){ + // Handle empty string + close(); + return; + }else { + buf = nextBuf; } + }else if(previousBuf !== undefined && nextBuf === undefined){ + buf = previousBuf; }else { - throw new Error(`Invalid Option: to must be an integer, got ${JSON.stringify(opts.to)}`); + buf = Buffer.concat([previousBuf, nextBuf]); } - } - // Normalize option `to_line` - if(options.to_line === undefined || options.to_line === null){ - options.to_line = -1; - }else { - if(typeof options.to_line === 'string' && /\d+/.test(options.to_line)){ - options.to_line = parseInt(options.to_line); - } - if(Number.isInteger(options.to_line)){ - if(options.to_line <= 0){ - throw new Error(`Invalid Option: to_line must be a positive integer greater than 0, got ${JSON.stringify(opts.to_line)}`); + // Handle UTF BOM + if(bomSkipped === false){ + if(bom === false){ + this.state.bomSkipped = true; + }else if(buf.length < 3){ + // No enough data + if(end === false){ + // Wait for more data + this.state.previousBuf = buf; + return; + } + }else { + for(const encoding in boms){ + if(boms[encoding].compare(buf, 0, boms[encoding].length) === 0){ + // Skip BOM + const bomLength = boms[encoding].length; + this.state.bufBytesStart += bomLength; + buf = buf.slice(bomLength); + // Renormalize original options with the new encoding + this.options = normalize_options({...this.original_options, encoding: encoding}); + break; + } + } + this.state.bomSkipped = true; } - }else { - throw new Error(`Invalid Option: to_line must be an integer, got ${JSON.stringify(opts.to_line)}`); } - } - this.info = { - bytes: 0, - comment_lines: 0, - empty_lines: 0, - invalid_field_length: 0, - lines: 1, - records: 0 - }; - this.options = options; - this.state = { - bomSkipped: false, - bufBytesStart: 0, - castField: fnCastField, - commenting: false, - // Current error encountered by a record - error: undefined, - enabled: options.from_line === 1, - escaping: false, - escapeIsQuote: isBuffer(options.escape) && isBuffer(options.quote) && Buffer.compare(options.escape, options.quote) === 0, - // columns can be `false`, `true`, `Array` - expectedRecordLength: Array.isArray(options.columns) ? options.columns.length : undefined, - field: new ResizeableBuffer(20), - firstLineToHeaders: fnFirstLineToHeaders, - needMoreDataSize: Math.max( - // Skip if the remaining buffer smaller than comment - options.comment !== null ? options.comment.length : 0, - // Skip if the remaining buffer can be delimiter - ...options.delimiter.map((delimiter) => delimiter.length), - // Skip if the remaining buffer can be escape sequence - options.quote !== null ? options.quote.length : 0, - ), - previousBuf: undefined, - quoting: false, - stop: false, - rawBuffer: new ResizeableBuffer(100), - record: [], - recordHasError: false, - record_length: 0, - recordDelimiterMaxLength: options.record_delimiter.length === 0 ? 2 : Math.max(...options.record_delimiter.map((v) => v.length)), - trimChars: [Buffer.from(' ', options.encoding)[0], Buffer.from('\t', options.encoding)[0]], - wasQuoting: false, - wasRowDelimiter: false - }; - } - // Implementation of `Transform._transform` - _transform(buf, encoding, callback){ - if(this.state.stop === true){ - return; - } - const err = this.__parse(buf, false); - if(err !== undefined){ - this.state.stop = true; - } - callback(err); - } - // Implementation of `Transform._flush` - _flush(callback){ - if(this.state.stop === true){ - return; - } - const err = this.__parse(undefined, true); - callback(err); - } - // Central parser implementation - __parse(nextBuf, end){ - const {bom, comment, escape, from_line, ltrim, max_record_size, quote, raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options; - let {record_delimiter} = this.options; - const {bomSkipped, previousBuf, rawBuffer, escapeIsQuote} = this.state; - let buf; - if(previousBuf === undefined){ - if(nextBuf === undefined){ - // Handle empty string - this.push(null); - return; - }else { - buf = nextBuf; - } - }else if(previousBuf !== undefined && nextBuf === undefined){ - buf = previousBuf; - }else { - buf = Buffer.concat([previousBuf, nextBuf]); - } - // Handle UTF BOM - if(bomSkipped === false){ - if(bom === false){ - this.state.bomSkipped = true; - }else if(buf.length < 3){ - // No enough data - if(end === false){ - // Wait for more data - this.state.previousBuf = buf; + const bufLen = buf.length; + let pos; + for(pos = 0; pos < bufLen; pos++){ + // Ensure we get enough space to look ahead + // There should be a way to move this out of the loop + if(this.__needMoreData(pos, bufLen, end)){ + break; + } + if(this.state.wasRowDelimiter === true){ + this.info.lines++; + this.state.wasRowDelimiter = false; + } + if(to_line !== -1 && this.info.lines > to_line){ + this.state.stop = true; + close(); return; } - }else { - for(const encoding in boms){ - if(boms[encoding].compare(buf, 0, boms[encoding].length) === 0){ - // Skip BOM - const bomLength = boms[encoding].length; - this.state.bufBytesStart += bomLength; - buf = buf.slice(bomLength); - // Renormalize original options with the new encoding - this.__normalizeOptions({...this.__originalOptions, encoding: encoding}); - break; + // Auto discovery of record_delimiter, unix, mac and windows supported + if(this.state.quoting === false && record_delimiter.length === 0){ + const record_delimiterCount = this.__autoDiscoverRecordDelimiter(buf, pos); + if(record_delimiterCount){ + record_delimiter = this.options.record_delimiter; } } - this.state.bomSkipped = true; - } - } - const bufLen = buf.length; - let pos; - for(pos = 0; pos < bufLen; pos++){ - // Ensure we get enough space to look ahead - // There should be a way to move this out of the loop - if(this.__needMoreData(pos, bufLen, end)){ - break; - } - if(this.state.wasRowDelimiter === true){ - this.info.lines++; - this.state.wasRowDelimiter = false; - } - if(to_line !== -1 && this.info.lines > to_line){ - this.state.stop = true; - this.push(null); - return; - } - // Auto discovery of record_delimiter, unix, mac and windows supported - if(this.state.quoting === false && record_delimiter.length === 0){ - const record_delimiterCount = this.__autoDiscoverRecordDelimiter(buf, pos); - if(record_delimiterCount){ - record_delimiter = this.options.record_delimiter; + const chr = buf[pos]; + if(raw === true){ + rawBuffer.append(chr); } - } - const chr = buf[pos]; - if(raw === true){ - rawBuffer.append(chr); - } - if((chr === cr || chr === nl) && this.state.wasRowDelimiter === false){ - this.state.wasRowDelimiter = true; - } - // Previous char was a valid escape char - // treat the current char as a regular char - if(this.state.escaping === true){ - this.state.escaping = false; - }else { - // Escape is only active inside quoted fields - // We are quoting, the char is an escape chr and there is a chr to escape - // if(escape !== null && this.state.quoting === true && chr === escape && pos + 1 < bufLen){ - if(escape !== null && this.state.quoting === true && this.__isEscape(buf, pos, chr) && pos + escape.length < bufLen){ - if(escapeIsQuote){ - if(this.__isQuote(buf, pos+escape.length)){ + if((chr === cr || chr === nl) && this.state.wasRowDelimiter === false){ + this.state.wasRowDelimiter = true; + } + // Previous char was a valid escape char + // treat the current char as a regular char + if(this.state.escaping === true){ + this.state.escaping = false; + }else { + // Escape is only active inside quoted fields + // We are quoting, the char is an escape chr and there is a chr to escape + // if(escape !== null && this.state.quoting === true && chr === escape && pos + 1 < bufLen){ + if(escape !== null && this.state.quoting === true && this.__isEscape(buf, pos, chr) && pos + escape.length < bufLen){ + if(escapeIsQuote){ + if(this.__isQuote(buf, pos+escape.length)){ + this.state.escaping = true; + pos += escape.length - 1; + continue; + } + }else { this.state.escaping = true; pos += escape.length - 1; continue; } - }else { - this.state.escaping = true; - pos += escape.length - 1; - continue; } - } - // Not currently escaping and chr is a quote - // TODO: need to compare bytes instead of single char - if(this.state.commenting === false && this.__isQuote(buf, pos)){ - if(this.state.quoting === true){ - const nextChr = buf[pos+quote.length]; - const isNextChrTrimable = rtrim && this.__isCharTrimable(nextChr); - const isNextChrComment = comment !== null && this.__compareBytes(comment, buf, pos+quote.length, nextChr); - const isNextChrDelimiter = this.__isDelimiter(buf, pos+quote.length, nextChr); - const isNextChrRecordDelimiter = record_delimiter.length === 0 ? this.__autoDiscoverRecordDelimiter(buf, pos+quote.length) : this.__isRecordDelimiter(nextChr, buf, pos+quote.length); - // Escape a quote - // Treat next char as a regular character - if(escape !== null && this.__isEscape(buf, pos, chr) && this.__isQuote(buf, pos + escape.length)){ - pos += escape.length - 1; - }else if(!nextChr || isNextChrDelimiter || isNextChrRecordDelimiter || isNextChrComment || isNextChrTrimable){ - this.state.quoting = false; - this.state.wasQuoting = true; - pos += quote.length - 1; - continue; - }else if(relax_quotes === false){ - const err = this.__error( - new CsvError('CSV_INVALID_CLOSING_QUOTE', [ - 'Invalid Closing Quote:', - `got "${String.fromCharCode(nextChr)}"`, - `at line ${this.info.lines}`, - 'instead of delimiter, record delimiter, trimable character', - '(if activated) or comment', - ], this.options, this.__infoField()) - ); - if(err !== undefined) return err; - }else { - this.state.quoting = false; - this.state.wasQuoting = true; - this.state.field.prepend(quote); - pos += quote.length - 1; - } - }else { - if(this.state.field.length !== 0){ - // In relax_quotes mode, treat opening quote preceded by chrs as regular - if(relax_quotes === false){ + // Not currently escaping and chr is a quote + // TODO: need to compare bytes instead of single char + if(this.state.commenting === false && this.__isQuote(buf, pos)){ + if(this.state.quoting === true){ + const nextChr = buf[pos+quote.length]; + const isNextChrTrimable = rtrim && this.__isCharTrimable(nextChr); + const isNextChrComment = comment !== null && this.__compareBytes(comment, buf, pos+quote.length, nextChr); + const isNextChrDelimiter = this.__isDelimiter(buf, pos+quote.length, nextChr); + const isNextChrRecordDelimiter = record_delimiter.length === 0 ? this.__autoDiscoverRecordDelimiter(buf, pos+quote.length) : this.__isRecordDelimiter(nextChr, buf, pos+quote.length); + // Escape a quote + // Treat next char as a regular character + if(escape !== null && this.__isEscape(buf, pos, chr) && this.__isQuote(buf, pos + escape.length)){ + pos += escape.length - 1; + }else if(!nextChr || isNextChrDelimiter || isNextChrRecordDelimiter || isNextChrComment || isNextChrTrimable){ + this.state.quoting = false; + this.state.wasQuoting = true; + pos += quote.length - 1; + continue; + }else if(relax_quotes === false){ const err = this.__error( - new CsvError('INVALID_OPENING_QUOTE', [ - 'Invalid Opening Quote:', - `a quote is found inside a field at line ${this.info.lines}`, - ], this.options, this.__infoField(), { - field: this.state.field, - }) + new CsvError('CSV_INVALID_CLOSING_QUOTE', [ + 'Invalid Closing Quote:', + `got "${String.fromCharCode(nextChr)}"`, + `at line ${this.info.lines}`, + 'instead of delimiter, record delimiter, trimable character', + '(if activated) or comment', + ], this.options, this.__infoField()) ); if(err !== undefined) return err; + }else { + this.state.quoting = false; + this.state.wasQuoting = true; + this.state.field.prepend(quote); + pos += quote.length - 1; } }else { - this.state.quoting = true; - pos += quote.length - 1; - continue; - } - } - } - if(this.state.quoting === false){ - const recordDelimiterLength = this.__isRecordDelimiter(chr, buf, pos); - if(recordDelimiterLength !== 0){ - // Do not emit comments which take a full line - const skipCommentLine = this.state.commenting && (this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0); - if(skipCommentLine){ - this.info.comment_lines++; - // Skip full comment line - }else { - // Activate records emition if above from_line - if(this.state.enabled === false && this.info.lines + (this.state.wasRowDelimiter === true ? 1: 0) >= from_line){ - this.state.enabled = true; - this.__resetField(); - this.__resetRecord(); - pos += recordDelimiterLength - 1; + if(this.state.field.length !== 0){ + // In relax_quotes mode, treat opening quote preceded by chrs as regular + if(relax_quotes === false){ + const err = this.__error( + new CsvError('INVALID_OPENING_QUOTE', [ + 'Invalid Opening Quote:', + `a quote is found inside a field at line ${this.info.lines}`, + ], this.options, this.__infoField(), { + field: this.state.field, + }) + ); + if(err !== undefined) return err; + } + }else { + this.state.quoting = true; + pos += quote.length - 1; continue; } - // Skip if line is empty and skip_empty_lines activated - if(skip_empty_lines === true && this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0){ - this.info.empty_lines++; - pos += recordDelimiterLength - 1; - continue; + } + } + if(this.state.quoting === false){ + const recordDelimiterLength = this.__isRecordDelimiter(chr, buf, pos); + if(recordDelimiterLength !== 0){ + // Do not emit comments which take a full line + const skipCommentLine = this.state.commenting && (this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0); + if(skipCommentLine){ + this.info.comment_lines++; + // Skip full comment line + }else { + // Activate records emition if above from_line + if(this.state.enabled === false && this.info.lines + (this.state.wasRowDelimiter === true ? 1: 0) >= from_line){ + this.state.enabled = true; + this.__resetField(); + this.__resetRecord(); + pos += recordDelimiterLength - 1; + continue; + } + // Skip if line is empty and skip_empty_lines activated + if(skip_empty_lines === true && this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0){ + this.info.empty_lines++; + pos += recordDelimiterLength - 1; + continue; + } + this.info.bytes = this.state.bufBytesStart + pos; + const errField = this.__onField(); + if(errField !== undefined) return errField; + this.info.bytes = this.state.bufBytesStart + pos + recordDelimiterLength; + const errRecord = this.__onRecord(push); + if(errRecord !== undefined) return errRecord; + if(to !== -1 && this.info.records >= to){ + this.state.stop = true; + close(); + return; + } } + this.state.commenting = false; + pos += recordDelimiterLength - 1; + continue; + } + if(this.state.commenting){ + continue; + } + const commentCount = comment === null ? 0 : this.__compareBytes(comment, buf, pos, chr); + if(commentCount !== 0){ + this.state.commenting = true; + continue; + } + const delimiterLength = this.__isDelimiter(buf, pos, chr); + if(delimiterLength !== 0){ this.info.bytes = this.state.bufBytesStart + pos; const errField = this.__onField(); if(errField !== undefined) return errField; - this.info.bytes = this.state.bufBytesStart + pos + recordDelimiterLength; - const errRecord = this.__onRecord(); - if(errRecord !== undefined) return errRecord; - if(to !== -1 && this.info.records >= to){ - this.state.stop = true; - this.push(null); - return; - } + pos += delimiterLength - 1; + continue; } - this.state.commenting = false; - pos += recordDelimiterLength - 1; - continue; - } - if(this.state.commenting){ - continue; } - const commentCount = comment === null ? 0 : this.__compareBytes(comment, buf, pos, chr); - if(commentCount !== 0){ - this.state.commenting = true; - continue; - } - const delimiterLength = this.__isDelimiter(buf, pos, chr); - if(delimiterLength !== 0){ - this.info.bytes = this.state.bufBytesStart + pos; - const errField = this.__onField(); - if(errField !== undefined) return errField; - pos += delimiterLength - 1; - continue; + } + if(this.state.commenting === false){ + if(max_record_size !== 0 && this.state.record_length + this.state.field.length > max_record_size){ + const err = this.__error( + new CsvError('CSV_MAX_RECORD_SIZE', [ + 'Max Record Size:', + 'record exceed the maximum number of tolerated bytes', + `of ${max_record_size}`, + `at line ${this.info.lines}`, + ], this.options, this.__infoField()) + ); + if(err !== undefined) return err; } } - } - if(this.state.commenting === false){ - if(max_record_size !== 0 && this.state.record_length + this.state.field.length > max_record_size){ + const lappend = ltrim === false || this.state.quoting === true || this.state.field.length !== 0 || !this.__isCharTrimable(chr); + // rtrim in non quoting is handle in __onField + const rappend = rtrim === false || this.state.wasQuoting === false; + if(lappend === true && rappend === true){ + this.state.field.append(chr); + }else if(rtrim === true && !this.__isCharTrimable(chr)){ const err = this.__error( - new CsvError('CSV_MAX_RECORD_SIZE', [ - 'Max Record Size:', - 'record exceed the maximum number of tolerated bytes', - `of ${max_record_size}`, + new CsvError('CSV_NON_TRIMABLE_CHAR_AFTER_CLOSING_QUOTE', [ + 'Invalid Closing Quote:', + 'found non trimable byte after quote', `at line ${this.info.lines}`, ], this.options, this.__infoField()) ); if(err !== undefined) return err; } } - const lappend = ltrim === false || this.state.quoting === true || this.state.field.length !== 0 || !this.__isCharTrimable(chr); - // rtrim in non quoting is handle in __onField - const rappend = rtrim === false || this.state.wasQuoting === false; - if(lappend === true && rappend === true){ - this.state.field.append(chr); - }else if(rtrim === true && !this.__isCharTrimable(chr)){ - const err = this.__error( - new CsvError('CSV_NON_TRIMABLE_CHAR_AFTER_CLOSING_QUOTE', [ - 'Invalid Closing Quote:', - 'found non trimable byte after quote', - `at line ${this.info.lines}`, - ], this.options, this.__infoField()) - ); - if(err !== undefined) return err; - } - } - if(end === true){ - // Ensure we are not ending in a quoting state - if(this.state.quoting === true){ - const err = this.__error( - new CsvError('CSV_QUOTE_NOT_CLOSED', [ - 'Quote Not Closed:', - `the parsing is finished with an opening quote at line ${this.info.lines}`, - ], this.options, this.__infoField()) - ); - if(err !== undefined) return err; + if(end === true){ + // Ensure we are not ending in a quoting state + if(this.state.quoting === true){ + const err = this.__error( + new CsvError('CSV_QUOTE_NOT_CLOSED', [ + 'Quote Not Closed:', + `the parsing is finished with an opening quote at line ${this.info.lines}`, + ], this.options, this.__infoField()) + ); + if(err !== undefined) return err; + }else { + // Skip last line if it has no characters + if(this.state.wasQuoting === true || this.state.record.length !== 0 || this.state.field.length !== 0){ + this.info.bytes = this.state.bufBytesStart + pos; + const errField = this.__onField(); + if(errField !== undefined) return errField; + const errRecord = this.__onRecord(push); + if(errRecord !== undefined) return errRecord; + }else if(this.state.wasRowDelimiter === true){ + this.info.empty_lines++; + }else if(this.state.commenting === true){ + this.info.comment_lines++; + } + } }else { - // Skip last line if it has no characters - if(this.state.wasQuoting === true || this.state.record.length !== 0 || this.state.field.length !== 0){ - this.info.bytes = this.state.bufBytesStart + pos; - const errField = this.__onField(); - if(errField !== undefined) return errField; - const errRecord = this.__onRecord(); - if(errRecord !== undefined) return errRecord; - }else if(this.state.wasRowDelimiter === true){ - this.info.empty_lines++; - }else if(this.state.commenting === true){ - this.info.comment_lines++; + this.state.bufBytesStart += pos; + this.state.previousBuf = buf.slice(pos); + } + if(this.state.wasRowDelimiter === true){ + this.info.lines++; + this.state.wasRowDelimiter = false; + } + }, + __onRecord: function(push){ + const {columns, group_columns_by_name, encoding, info, from, relax_column_count, relax_column_count_less, relax_column_count_more, raw, skip_records_with_empty_values} = this.options; + const {enabled, record} = this.state; + if(enabled === false){ + return this.__resetRecord(); + } + // Convert the first line into column names + const recordLength = record.length; + if(columns === true){ + if(skip_records_with_empty_values === true && isRecordEmpty(record)){ + this.__resetRecord(); + return; + } + return this.__firstLineToColumns(record); + } + if(columns === false && this.info.records === 0){ + this.state.expectedRecordLength = recordLength; + } + if(recordLength !== this.state.expectedRecordLength){ + const err = columns === false ? + new CsvError('CSV_RECORD_INCONSISTENT_FIELDS_LENGTH', [ + 'Invalid Record Length:', + `expect ${this.state.expectedRecordLength},`, + `got ${recordLength} on line ${this.info.lines}`, + ], this.options, this.__infoField(), { + record: record, + }) + : + new CsvError('CSV_RECORD_INCONSISTENT_COLUMNS', [ + 'Invalid Record Length:', + `columns length is ${columns.length},`, // rename columns + `got ${recordLength} on line ${this.info.lines}`, + ], this.options, this.__infoField(), { + record: record, + }); + if(relax_column_count === true || + (relax_column_count_less === true && recordLength < this.state.expectedRecordLength) || + (relax_column_count_more === true && recordLength > this.state.expectedRecordLength)){ + this.info.invalid_field_length++; + this.state.error = err; + // Error is undefined with skip_records_with_error + }else { + const finalErr = this.__error(err); + if(finalErr) return finalErr; } } - }else { - this.state.bufBytesStart += pos; - this.state.previousBuf = buf.slice(pos); - } - if(this.state.wasRowDelimiter === true){ - this.info.lines++; - this.state.wasRowDelimiter = false; - } - } - __onRecord(){ - const {columns, group_columns_by_name, encoding, info, from, relax_column_count, relax_column_count_less, relax_column_count_more, raw, skip_records_with_empty_values} = this.options; - const {enabled, record} = this.state; - if(enabled === false){ - return this.__resetRecord(); - } - // Convert the first line into column names - const recordLength = record.length; - if(columns === true){ if(skip_records_with_empty_values === true && isRecordEmpty(record)){ this.__resetRecord(); return; } - return this.__firstLineToColumns(record); - } - if(columns === false && this.info.records === 0){ - this.state.expectedRecordLength = recordLength; - } - if(recordLength !== this.state.expectedRecordLength){ - const err = columns === false ? - new CsvError('CSV_RECORD_INCONSISTENT_FIELDS_LENGTH', [ - 'Invalid Record Length:', - `expect ${this.state.expectedRecordLength},`, - `got ${recordLength} on line ${this.info.lines}`, - ], this.options, this.__infoField(), { - record: record, - }) - : - new CsvError('CSV_RECORD_INCONSISTENT_COLUMNS', [ - 'Invalid Record Length:', - `columns length is ${columns.length},`, // rename columns - `got ${recordLength} on line ${this.info.lines}`, - ], this.options, this.__infoField(), { - record: record, - }); - if(relax_column_count === true || - (relax_column_count_less === true && recordLength < this.state.expectedRecordLength) || - (relax_column_count_more === true && recordLength > this.state.expectedRecordLength)){ - this.info.invalid_field_length++; - this.state.error = err; - // Error is undefined with skip_records_with_error - }else { - const finalErr = this.__error(err); - if(finalErr) return finalErr; + if(this.state.recordHasError === true){ + this.__resetRecord(); + this.state.recordHasError = false; + return; } - } - if(skip_records_with_empty_values === true && isRecordEmpty(record)){ - this.__resetRecord(); - return; - } - if(this.state.recordHasError === true){ - this.__resetRecord(); - this.state.recordHasError = false; - return; - } - this.info.records++; - if(from === 1 || this.info.records >= from){ - const {objname} = this.options; - // With columns, records are object - if(columns !== false){ - const obj = {}; - // Transform record array to an object - for(let i = 0, l = record.length; i < l; i++){ - if(columns[i] === undefined || columns[i].disabled) continue; - // Turn duplicate columns into an array - if (group_columns_by_name === true && obj[columns[i].name] !== undefined) { - if (Array.isArray(obj[columns[i].name])) { - obj[columns[i].name] = obj[columns[i].name].concat(record[i]); + this.info.records++; + if(from === 1 || this.info.records >= from){ + const {objname} = this.options; + // With columns, records are object + if(columns !== false){ + const obj = {}; + // Transform record array to an object + for(let i = 0, l = record.length; i < l; i++){ + if(columns[i] === undefined || columns[i].disabled) continue; + // Turn duplicate columns into an array + if (group_columns_by_name === true && obj[columns[i].name] !== undefined) { + if (Array.isArray(obj[columns[i].name])) { + obj[columns[i].name] = obj[columns[i].name].concat(record[i]); + } else { + obj[columns[i].name] = [obj[columns[i].name], record[i]]; + } } else { - obj[columns[i].name] = [obj[columns[i].name], record[i]]; + obj[columns[i].name] = record[i]; } - } else { - obj[columns[i].name] = record[i]; } - } - // Without objname (default) - if(raw === true || info === true){ - const extRecord = Object.assign( - {record: obj}, - (raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}), - (info === true ? {info: this.__infoRecord()}: {}) - ); - const err = this.__push( - objname === undefined ? extRecord : [obj[objname], extRecord] - ); - if(err){ - return err; - } - }else { - const err = this.__push( - objname === undefined ? obj : [obj[objname], obj] - ); - if(err){ - return err; - } - } - // Without columns, records are array - }else { - if(raw === true || info === true){ - const extRecord = Object.assign( - {record: record}, - raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}, - info === true ? {info: this.__infoRecord()}: {} - ); - const err = this.__push( - objname === undefined ? extRecord : [record[objname], extRecord] - ); - if(err){ - return err; + // Without objname (default) + if(raw === true || info === true){ + const extRecord = Object.assign( + {record: obj}, + (raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}), + (info === true ? {info: this.__infoRecord()}: {}) + ); + const err = this.__push( + objname === undefined ? extRecord : [obj[objname], extRecord] + , push); + if(err){ + return err; + } + }else { + const err = this.__push( + objname === undefined ? obj : [obj[objname], obj] + , push); + if(err){ + return err; + } } + // Without columns, records are array }else { - const err = this.__push( - objname === undefined ? record : [record[objname], record] - ); - if(err){ - return err; + if(raw === true || info === true){ + const extRecord = Object.assign( + {record: record}, + raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}, + info === true ? {info: this.__infoRecord()}: {} + ); + const err = this.__push( + objname === undefined ? extRecord : [record[objname], extRecord] + , push); + if(err){ + return err; + } + }else { + const err = this.__push( + objname === undefined ? record : [record[objname], record] + , push); + if(err){ + return err; + } } } } - } - this.__resetRecord(); - } - __firstLineToColumns(record){ - const {firstLineToHeaders} = this.state; - try{ - const headers = firstLineToHeaders === undefined ? record : firstLineToHeaders.call(null, record); - if(!Array.isArray(headers)){ - return this.__error( - new CsvError('CSV_INVALID_COLUMN_MAPPING', [ - 'Invalid Column Mapping:', - 'expect an array from column function,', - `got ${JSON.stringify(headers)}` - ], this.options, this.__infoField(), { - headers: headers, - }) - ); - } - const normalizedHeaders = normalizeColumnsArray(headers); - this.state.expectedRecordLength = normalizedHeaders.length; - this.options.columns = normalizedHeaders; this.__resetRecord(); - return; - }catch(err){ - return err; - } - } - __resetRecord(){ - if(this.options.raw === true){ - this.state.rawBuffer.reset(); - } - this.state.error = undefined; - this.state.record = []; - this.state.record_length = 0; - } - __onField(){ - const {cast, encoding, rtrim, max_record_size} = this.options; - const {enabled, wasQuoting} = this.state; - // Short circuit for the from_line options - if(enabled === false){ - return this.__resetField(); - } - let field = this.state.field.toString(encoding); - if(rtrim === true && wasQuoting === false){ - field = field.trimRight(); - } - if(cast === true){ - const [err, f] = this.__cast(field); - if(err !== undefined) return err; - field = f; - } - this.state.record.push(field); - // Increment record length if record size must not exceed a limit - if(max_record_size !== 0 && typeof field === 'string'){ - this.state.record_length += field.length; - } - this.__resetField(); - } - __resetField(){ - this.state.field.reset(); - this.state.wasQuoting = false; - } - __push(record){ - const {on_record} = this.options; - if(on_record !== undefined){ - const info = this.__infoRecord(); + }, + __firstLineToColumns: function(record){ + const {firstLineToHeaders} = this.state; try{ - record = on_record.call(null, record, info); + const headers = firstLineToHeaders === undefined ? record : firstLineToHeaders.call(null, record); + if(!Array.isArray(headers)){ + return this.__error( + new CsvError('CSV_INVALID_COLUMN_MAPPING', [ + 'Invalid Column Mapping:', + 'expect an array from column function,', + `got ${JSON.stringify(headers)}` + ], this.options, this.__infoField(), { + headers: headers, + }) + ); + } + const normalizedHeaders = normalize_columns_array(headers); + this.state.expectedRecordLength = normalizedHeaders.length; + this.options.columns = normalizedHeaders; + this.__resetRecord(); + return; }catch(err){ return err; } - if(record === undefined || record === null){ return; } - } - this.push(record); - } - // Return a tuple with the error and the casted value - __cast(field){ - const {columns, relax_column_count} = this.options; - const isColumns = Array.isArray(columns); - // Dont loose time calling cast - // because the final record is an object - // and this field can't be associated to a key present in columns - if(isColumns === true && relax_column_count && this.options.columns.length <= this.state.record.length){ - return [undefined, undefined]; - } - if(this.state.castField !== null){ - try{ + }, + __resetRecord: function(){ + if(this.options.raw === true){ + this.state.rawBuffer.reset(); + } + this.state.error = undefined; + this.state.record = []; + this.state.record_length = 0; + }, + __onField: function(){ + const {cast, encoding, rtrim, max_record_size} = this.options; + const {enabled, wasQuoting} = this.state; + // Short circuit for the from_line options + if(enabled === false){ + return this.__resetField(); + } + let field = this.state.field.toString(encoding); + if(rtrim === true && wasQuoting === false){ + field = field.trimRight(); + } + if(cast === true){ + const [err, f] = this.__cast(field); + if(err !== undefined) return err; + field = f; + } + this.state.record.push(field); + // Increment record length if record size must not exceed a limit + if(max_record_size !== 0 && typeof field === 'string'){ + this.state.record_length += field.length; + } + this.__resetField(); + }, + __resetField: function(){ + this.state.field.reset(); + this.state.wasQuoting = false; + }, + __push: function(record, push){ + const {on_record} = this.options; + if(on_record !== undefined){ + const info = this.__infoRecord(); + try{ + record = on_record.call(null, record, info); + }catch(err){ + return err; + } + if(record === undefined || record === null){ return; } + } + push(record); + }, + // Return a tuple with the error and the casted value + __cast: function(field){ + const {columns, relax_column_count} = this.options; + const isColumns = Array.isArray(columns); + // Dont loose time calling cast + // because the final record is an object + // and this field can't be associated to a key present in columns + if(isColumns === true && relax_column_count && this.options.columns.length <= this.state.record.length){ + return [undefined, undefined]; + } + if(this.state.castField !== null){ + try{ + const info = this.__infoField(); + return [undefined, this.state.castField.call(null, field, info)]; + }catch(err){ + return [err]; + } + } + if(this.__isFloat(field)){ + return [undefined, parseFloat(field)]; + }else if(this.options.cast_date !== false){ const info = this.__infoField(); - return [undefined, this.state.castField.call(null, field, info)]; - }catch(err){ - return [err]; + return [undefined, this.options.cast_date.call(null, field, info)]; } - } - if(this.__isFloat(field)){ - return [undefined, parseFloat(field)]; - }else if(this.options.cast_date !== false){ - const info = this.__infoField(); - return [undefined, this.options.cast_date.call(null, field, info)]; - } - return [undefined, field]; - } - // Helper to test if a character is a space or a line delimiter - __isCharTrimable(chr){ - return chr === space || chr === tab || chr === cr || chr === nl || chr === np; - } - // Keep it in case we implement the `cast_int` option - // __isInt(value){ - // // return Number.isInteger(parseInt(value)) - // // return !isNaN( parseInt( obj ) ); - // return /^(\-|\+)?[1-9][0-9]*$/.test(value) - // } - __isFloat(value){ - return (value - parseFloat(value) + 1) >= 0; // Borrowed from jquery - } - __compareBytes(sourceBuf, targetBuf, targetPos, firstByte){ - if(sourceBuf[0] !== firstByte) return 0; - const sourceLength = sourceBuf.length; - for(let i = 1; i < sourceLength; i++){ - if(sourceBuf[i] !== targetBuf[targetPos+i]) return 0; - } - return sourceLength; - } - __needMoreData(i, bufLen, end){ - if(end) return false; - const {quote} = this.options; - const {quoting, needMoreDataSize, recordDelimiterMaxLength} = this.state; - const numOfCharLeft = bufLen - i - 1; - const requiredLength = Math.max( - needMoreDataSize, - // Skip if the remaining buffer smaller than record delimiter - recordDelimiterMaxLength, - // Skip if the remaining buffer can be record delimiter following the closing quote - // 1 is for quote.length - quoting ? (quote.length + recordDelimiterMaxLength) : 0, - ); - return numOfCharLeft < requiredLength; - } - __isDelimiter(buf, pos, chr){ - const {delimiter, ignore_last_delimiters} = this.options; - if(ignore_last_delimiters === true && this.state.record.length === this.options.columns.length - 1){ - return 0; - }else if(ignore_last_delimiters !== false && typeof ignore_last_delimiters === 'number' && this.state.record.length === ignore_last_delimiters - 1){ - return 0; - } - loop1: for(let i = 0; i < delimiter.length; i++){ - const del = delimiter[i]; - if(del[0] === chr){ - for(let j = 1; j < del.length; j++){ - if(del[j] !== buf[pos+j]) continue loop1; + return [undefined, field]; + }, + // Helper to test if a character is a space or a line delimiter + __isCharTrimable: function(chr){ + return chr === space || chr === tab || chr === cr || chr === nl || chr === np; + }, + // Keep it in case we implement the `cast_int` option + // __isInt(value){ + // // return Number.isInteger(parseInt(value)) + // // return !isNaN( parseInt( obj ) ); + // return /^(\-|\+)?[1-9][0-9]*$/.test(value) + // } + __isFloat: function(value){ + return (value - parseFloat(value) + 1) >= 0; // Borrowed from jquery + }, + __compareBytes: function(sourceBuf, targetBuf, targetPos, firstByte){ + if(sourceBuf[0] !== firstByte) return 0; + const sourceLength = sourceBuf.length; + for(let i = 1; i < sourceLength; i++){ + if(sourceBuf[i] !== targetBuf[targetPos+i]) return 0; + } + return sourceLength; + }, + __isDelimiter: function(buf, pos, chr){ + const {delimiter, ignore_last_delimiters} = this.options; + if(ignore_last_delimiters === true && this.state.record.length === this.options.columns.length - 1){ + return 0; + }else if(ignore_last_delimiters !== false && typeof ignore_last_delimiters === 'number' && this.state.record.length === ignore_last_delimiters - 1){ + return 0; + } + loop1: for(let i = 0; i < delimiter.length; i++){ + const del = delimiter[i]; + if(del[0] === chr){ + for(let j = 1; j < del.length; j++){ + if(del[j] !== buf[pos+j]) continue loop1; + } + return del.length; } - return del.length; } - } - return 0; - } - __isRecordDelimiter(chr, buf, pos){ - const {record_delimiter} = this.options; - const recordDelimiterLength = record_delimiter.length; - loop1: for(let i = 0; i < recordDelimiterLength; i++){ - const rd = record_delimiter[i]; - const rdLength = rd.length; - if(rd[0] !== chr){ - continue; + return 0; + }, + __isRecordDelimiter: function(chr, buf, pos){ + const {record_delimiter} = this.options; + const recordDelimiterLength = record_delimiter.length; + loop1: for(let i = 0; i < recordDelimiterLength; i++){ + const rd = record_delimiter[i]; + const rdLength = rd.length; + if(rd[0] !== chr){ + continue; + } + for(let j = 1; j < rdLength; j++){ + if(rd[j] !== buf[pos+j]){ + continue loop1; + } + } + return rd.length; } - for(let j = 1; j < rdLength; j++){ - if(rd[j] !== buf[pos+j]){ - continue loop1; + return 0; + }, + __isEscape: function(buf, pos, chr){ + const {escape} = this.options; + if(escape === null) return false; + const l = escape.length; + if(escape[0] === chr){ + for(let i = 0; i < l; i++){ + if(escape[i] !== buf[pos+i]){ + return false; + } } + return true; } - return rd.length; - } - return 0; - } - __isEscape(buf, pos, chr){ - const {escape} = this.options; - if(escape === null) return false; - const l = escape.length; - if(escape[0] === chr){ + return false; + }, + __isQuote: function(buf, pos){ + const {quote} = this.options; + if(quote === null) return false; + const l = quote.length; for(let i = 0; i < l; i++){ - if(escape[i] !== buf[pos+i]){ + if(quote[i] !== buf[pos+i]){ return false; } } return true; - } - return false; - } - __isQuote(buf, pos){ - const {quote} = this.options; - if(quote === null) return false; - const l = quote.length; - for(let i = 0; i < l; i++){ - if(quote[i] !== buf[pos+i]){ - return false; - } - } - return true; - } - __autoDiscoverRecordDelimiter(buf, pos){ - const {encoding} = this.options; - const chr = buf[pos]; - if(chr === cr){ - if(buf[pos+1] === nl){ - this.options.record_delimiter.push(Buffer.from('\r\n', encoding)); - this.state.recordDelimiterMaxLength = 2; - return 2; - }else { - this.options.record_delimiter.push(Buffer.from('\r', encoding)); + }, + __autoDiscoverRecordDelimiter: function(buf, pos){ + const {encoding} = this.options; + const chr = buf[pos]; + if(chr === cr){ + if(buf[pos+1] === nl){ + this.options.record_delimiter.push(Buffer.from('\r\n', encoding)); + this.state.recordDelimiterMaxLength = 2; + return 2; + }else { + this.options.record_delimiter.push(Buffer.from('\r', encoding)); + this.state.recordDelimiterMaxLength = 1; + return 1; + } + }else if(chr === nl){ + this.options.record_delimiter.push(Buffer.from('\n', encoding)); this.state.recordDelimiterMaxLength = 1; return 1; } - }else if(chr === nl){ - this.options.record_delimiter.push(Buffer.from('\n', encoding)); - this.state.recordDelimiterMaxLength = 1; - return 1; - } - return 0; - } - __error(msg){ - const {encoding, raw, skip_records_with_error} = this.options; - const err = typeof msg === 'string' ? new Error(msg) : msg; - if(skip_records_with_error){ - this.state.recordHasError = true; - this.emit('skip', err, raw ? this.state.rawBuffer.toString(encoding) : undefined); - return undefined; - }else { - return err; + return 0; + }, + __error: function(msg){ + const {encoding, raw, skip_records_with_error} = this.options; + const err = typeof msg === 'string' ? new Error(msg) : msg; + if(skip_records_with_error){ + this.state.recordHasError = true; + if(this.options.on_skip !== undefined){ + this.options.on_skip(err, raw ? this.state.rawBuffer.toString(encoding) : undefined); + } + // this.emit('skip', err, raw ? this.state.rawBuffer.toString(encoding) : undefined); + return undefined; + }else { + return err; + } + }, + __infoDataSet: function(){ + return { + ...this.info, + columns: this.options.columns + }; + }, + __infoRecord: function(){ + const {columns, raw, encoding} = this.options; + return { + ...this.__infoDataSet(), + error: this.state.error, + header: columns === true, + index: this.state.record.length, + raw: raw ? this.state.rawBuffer.toString(encoding) : undefined + }; + }, + __infoField: function(){ + const {columns} = this.options; + const isColumns = Array.isArray(columns); + return { + ...this.__infoRecord(), + column: isColumns === true ? + (columns.length > this.state.record.length ? + columns[this.state.record.length].name : + null + ) : + this.state.record.length, + quoting: this.state.wasQuoting, + }; } - } - __infoDataSet(){ - return { - ...this.info, - columns: this.options.columns + }; +}; + +class Parser extends Transform { + constructor(opts = {}){ + super({...{readableObjectMode: true}, ...opts, encoding: null}); + this.api = transform(opts); + this.api.options.on_skip = (err, chunk) => { + this.emit('skip', err, chunk); }; + // Backward compatibility + this.state = this.api.state; + this.options = this.api.options; + this.info = this.api.info; } - __infoRecord(){ - const {columns, raw, encoding} = this.options; - return { - ...this.__infoDataSet(), - error: this.state.error, - header: columns === true, - index: this.state.record.length, - raw: raw ? this.state.rawBuffer.toString(encoding) : undefined - }; + // Implementation of `Transform._transform` + _transform(buf, encoding, callback){ + if(this.state.stop === true){ + return; + } + const err = this.api.parse(buf, false, (record) => { + this.push.call(this, record); + }, () => { + this.push.call(this, null); + }); + if(err !== undefined){ + this.state.stop = true; + } + callback(err); } - __infoField(){ - const {columns} = this.options; - const isColumns = Array.isArray(columns); - return { - ...this.__infoRecord(), - column: isColumns === true ? - (columns.length > this.state.record.length ? - columns[this.state.record.length].name : - null - ) : - this.state.record.length, - quoting: this.state.wasQuoting, - }; + // Implementation of `Transform._flush` + _flush(callback){ + if(this.state.stop === true){ + return; + } + const err = this.api.parse(undefined, true, (record) => { + this.push.call(this, record); + }, () => { + this.push.call(this, null); + }); + callback(err); } } @@ -6242,7 +6273,7 @@ const parse = function(){ const type = typeof argument; if(data === undefined && (typeof argument === 'string' || isBuffer(argument))){ data = argument; - }else if(options === undefined && isObject(argument)){ + }else if(options === undefined && is_object(argument)){ options = argument; }else if(callback === undefined && type === 'function'){ callback = argument; @@ -6267,10 +6298,10 @@ const parse = function(){ } }); parser.on('error', function(err){ - callback(err, undefined, parser.__infoDataSet()); + callback(err, undefined, parser.api.__infoDataSet()); }); parser.on('end', function(){ - callback(undefined, records, parser.__infoDataSet()); + callback(undefined, records, parser.api.__infoDataSet()); }); } if(data !== undefined){ diff --git a/packages/csv-parse/dist/esm/sync.js b/packages/csv-parse/dist/esm/sync.js index dff3f20b1..cdef53688 100644 --- a/packages/csv-parse/dist/esm/sync.js +++ b/packages/csv-parse/dist/esm/sync.js @@ -199,7 +199,7 @@ function write (buffer, value, offset, isLE, mLen, nBytes) { var toString = {}.toString; -var isArray$1 = Array.isArray || function (arr) { +var isArray = Array.isArray || function (arr) { return toString.call(arr) == '[object Array]'; }; @@ -467,7 +467,7 @@ function fromObject (that, obj) { return fromArrayLike(that, obj) } - if (obj.type === 'Buffer' && isArray$1(obj.data)) { + if (obj.type === 'Buffer' && isArray(obj.data)) { return fromArrayLike(that, obj.data) } } @@ -532,7 +532,7 @@ Buffer.isEncoding = function isEncoding (encoding) { }; Buffer.concat = function concat (list, length) { - if (!isArray$1(list)) { + if (!isArray(list)) { throw new TypeError('"list" argument must be an Array of Buffers') } @@ -1963,3007 +1963,53 @@ function isSlowBuffer (obj) { return typeof obj.readFloatLE === 'function' && typeof obj.slice === 'function' && isFastBuffer(obj.slice(0, 0)) } -var domain; - -// This constructor is used to store event handlers. Instantiating this is -// faster than explicitly calling `Object.create(null)` to get a "clean" empty -// object (tested with v8 v4.9). -function EventHandlers() {} -EventHandlers.prototype = Object.create(null); - -function EventEmitter() { - EventEmitter.init.call(this); -} - -// nodejs oddity -// require('events') === require('events').EventEmitter -EventEmitter.EventEmitter = EventEmitter; - -EventEmitter.usingDomains = false; - -EventEmitter.prototype.domain = undefined; -EventEmitter.prototype._events = undefined; -EventEmitter.prototype._maxListeners = undefined; - -// By default EventEmitters will print a warning if more than 10 listeners are -// added to it. This is a useful default which helps finding memory leaks. -EventEmitter.defaultMaxListeners = 10; - -EventEmitter.init = function() { - this.domain = null; - if (EventEmitter.usingDomains) { - // if there is an active domain, then attach to it. - if (domain.active ) ; - } - - if (!this._events || this._events === Object.getPrototypeOf(this)._events) { - this._events = new EventHandlers(); - this._eventsCount = 0; - } - - this._maxListeners = this._maxListeners || undefined; -}; - -// Obviously not all Emitters should be limited to 10. This function allows -// that to be increased. Set to zero for unlimited. -EventEmitter.prototype.setMaxListeners = function setMaxListeners(n) { - if (typeof n !== 'number' || n < 0 || isNaN(n)) - throw new TypeError('"n" argument must be a positive number'); - this._maxListeners = n; - return this; -}; - -function $getMaxListeners(that) { - if (that._maxListeners === undefined) - return EventEmitter.defaultMaxListeners; - return that._maxListeners; -} - -EventEmitter.prototype.getMaxListeners = function getMaxListeners() { - return $getMaxListeners(this); -}; - -// These standalone emit* functions are used to optimize calling of event -// handlers for fast cases because emit() itself often has a variable number of -// arguments and can be deoptimized because of that. These functions always have -// the same number of arguments and thus do not get deoptimized, so the code -// inside them can execute faster. -function emitNone(handler, isFn, self) { - if (isFn) - handler.call(self); - else { - var len = handler.length; - var listeners = arrayClone(handler, len); - for (var i = 0; i < len; ++i) - listeners[i].call(self); - } -} -function emitOne(handler, isFn, self, arg1) { - if (isFn) - handler.call(self, arg1); - else { - var len = handler.length; - var listeners = arrayClone(handler, len); - for (var i = 0; i < len; ++i) - listeners[i].call(self, arg1); - } -} -function emitTwo(handler, isFn, self, arg1, arg2) { - if (isFn) - handler.call(self, arg1, arg2); - else { - var len = handler.length; - var listeners = arrayClone(handler, len); - for (var i = 0; i < len; ++i) - listeners[i].call(self, arg1, arg2); - } -} -function emitThree(handler, isFn, self, arg1, arg2, arg3) { - if (isFn) - handler.call(self, arg1, arg2, arg3); - else { - var len = handler.length; - var listeners = arrayClone(handler, len); - for (var i = 0; i < len; ++i) - listeners[i].call(self, arg1, arg2, arg3); - } -} - -function emitMany(handler, isFn, self, args) { - if (isFn) - handler.apply(self, args); - else { - var len = handler.length; - var listeners = arrayClone(handler, len); - for (var i = 0; i < len; ++i) - listeners[i].apply(self, args); - } -} - -EventEmitter.prototype.emit = function emit(type) { - var er, handler, len, args, i, events, domain; - var doError = (type === 'error'); - - events = this._events; - if (events) - doError = (doError && events.error == null); - else if (!doError) - return false; - - domain = this.domain; - - // If there is no 'error' event listener then throw. - if (doError) { - er = arguments[1]; - if (domain) { - if (!er) - er = new Error('Uncaught, unspecified "error" event'); - er.domainEmitter = this; - er.domain = domain; - er.domainThrown = false; - domain.emit('error', er); - } else if (er instanceof Error) { - throw er; // Unhandled 'error' event - } else { - // At least give some kind of context to the user - var err = new Error('Uncaught, unspecified "error" event. (' + er + ')'); - err.context = er; - throw err; - } - return false; - } - - handler = events[type]; - - if (!handler) - return false; - - var isFn = typeof handler === 'function'; - len = arguments.length; - switch (len) { - // fast cases - case 1: - emitNone(handler, isFn, this); - break; - case 2: - emitOne(handler, isFn, this, arguments[1]); - break; - case 3: - emitTwo(handler, isFn, this, arguments[1], arguments[2]); - break; - case 4: - emitThree(handler, isFn, this, arguments[1], arguments[2], arguments[3]); - break; - // slower - default: - args = new Array(len - 1); - for (i = 1; i < len; i++) - args[i - 1] = arguments[i]; - emitMany(handler, isFn, this, args); - } - - return true; -}; - -function _addListener(target, type, listener, prepend) { - var m; - var events; - var existing; - - if (typeof listener !== 'function') - throw new TypeError('"listener" argument must be a function'); - - events = target._events; - if (!events) { - events = target._events = new EventHandlers(); - target._eventsCount = 0; - } else { - // To avoid recursion in the case that type === "newListener"! Before - // adding it to the listeners, first emit "newListener". - if (events.newListener) { - target.emit('newListener', type, - listener.listener ? listener.listener : listener); - - // Re-assign `events` because a newListener handler could have caused the - // this._events to be assigned to a new object - events = target._events; - } - existing = events[type]; - } - - if (!existing) { - // Optimize the case of one listener. Don't need the extra array object. - existing = events[type] = listener; - ++target._eventsCount; - } else { - if (typeof existing === 'function') { - // Adding the second element, need to change to array. - existing = events[type] = prepend ? [listener, existing] : - [existing, listener]; - } else { - // If we've already got an array, just append. - if (prepend) { - existing.unshift(listener); - } else { - existing.push(listener); - } - } - - // Check for listener leak - if (!existing.warned) { - m = $getMaxListeners(target); - if (m && m > 0 && existing.length > m) { - existing.warned = true; - var w = new Error('Possible EventEmitter memory leak detected. ' + - existing.length + ' ' + type + ' listeners added. ' + - 'Use emitter.setMaxListeners() to increase limit'); - w.name = 'MaxListenersExceededWarning'; - w.emitter = target; - w.type = type; - w.count = existing.length; - emitWarning(w); - } - } - } - - return target; -} -function emitWarning(e) { - typeof console.warn === 'function' ? console.warn(e) : console.log(e); -} -EventEmitter.prototype.addListener = function addListener(type, listener) { - return _addListener(this, type, listener, false); -}; - -EventEmitter.prototype.on = EventEmitter.prototype.addListener; - -EventEmitter.prototype.prependListener = - function prependListener(type, listener) { - return _addListener(this, type, listener, true); - }; - -function _onceWrap(target, type, listener) { - var fired = false; - function g() { - target.removeListener(type, g); - if (!fired) { - fired = true; - listener.apply(target, arguments); - } - } - g.listener = listener; - return g; -} - -EventEmitter.prototype.once = function once(type, listener) { - if (typeof listener !== 'function') - throw new TypeError('"listener" argument must be a function'); - this.on(type, _onceWrap(this, type, listener)); - return this; -}; - -EventEmitter.prototype.prependOnceListener = - function prependOnceListener(type, listener) { - if (typeof listener !== 'function') - throw new TypeError('"listener" argument must be a function'); - this.prependListener(type, _onceWrap(this, type, listener)); - return this; - }; - -// emits a 'removeListener' event iff the listener was removed -EventEmitter.prototype.removeListener = - function removeListener(type, listener) { - var list, events, position, i, originalListener; - - if (typeof listener !== 'function') - throw new TypeError('"listener" argument must be a function'); - - events = this._events; - if (!events) - return this; - - list = events[type]; - if (!list) - return this; - - if (list === listener || (list.listener && list.listener === listener)) { - if (--this._eventsCount === 0) - this._events = new EventHandlers(); - else { - delete events[type]; - if (events.removeListener) - this.emit('removeListener', type, list.listener || listener); - } - } else if (typeof list !== 'function') { - position = -1; - - for (i = list.length; i-- > 0;) { - if (list[i] === listener || - (list[i].listener && list[i].listener === listener)) { - originalListener = list[i].listener; - position = i; - break; - } - } - - if (position < 0) - return this; - - if (list.length === 1) { - list[0] = undefined; - if (--this._eventsCount === 0) { - this._events = new EventHandlers(); - return this; - } else { - delete events[type]; - } - } else { - spliceOne(list, position); - } - - if (events.removeListener) - this.emit('removeListener', type, originalListener || listener); - } - - return this; - }; - -EventEmitter.prototype.removeAllListeners = - function removeAllListeners(type) { - var listeners, events; - - events = this._events; - if (!events) - return this; - - // not listening for removeListener, no need to emit - if (!events.removeListener) { - if (arguments.length === 0) { - this._events = new EventHandlers(); - this._eventsCount = 0; - } else if (events[type]) { - if (--this._eventsCount === 0) - this._events = new EventHandlers(); - else - delete events[type]; - } - return this; - } - - // emit removeListener for all listeners on all events - if (arguments.length === 0) { - var keys = Object.keys(events); - for (var i = 0, key; i < keys.length; ++i) { - key = keys[i]; - if (key === 'removeListener') continue; - this.removeAllListeners(key); - } - this.removeAllListeners('removeListener'); - this._events = new EventHandlers(); - this._eventsCount = 0; - return this; - } - - listeners = events[type]; - - if (typeof listeners === 'function') { - this.removeListener(type, listeners); - } else if (listeners) { - // LIFO order - do { - this.removeListener(type, listeners[listeners.length - 1]); - } while (listeners[0]); - } - - return this; - }; - -EventEmitter.prototype.listeners = function listeners(type) { - var evlistener; - var ret; - var events = this._events; - - if (!events) - ret = []; - else { - evlistener = events[type]; - if (!evlistener) - ret = []; - else if (typeof evlistener === 'function') - ret = [evlistener.listener || evlistener]; - else - ret = unwrapListeners(evlistener); - } - - return ret; -}; - -EventEmitter.listenerCount = function(emitter, type) { - if (typeof emitter.listenerCount === 'function') { - return emitter.listenerCount(type); - } else { - return listenerCount$1.call(emitter, type); - } -}; - -EventEmitter.prototype.listenerCount = listenerCount$1; -function listenerCount$1(type) { - var events = this._events; - - if (events) { - var evlistener = events[type]; - - if (typeof evlistener === 'function') { - return 1; - } else if (evlistener) { - return evlistener.length; - } - } - - return 0; -} - -EventEmitter.prototype.eventNames = function eventNames() { - return this._eventsCount > 0 ? Reflect.ownKeys(this._events) : []; -}; - -// About 1.5x faster than the two-arg version of Array#splice(). -function spliceOne(list, index) { - for (var i = index, k = i + 1, n = list.length; k < n; i += 1, k += 1) - list[i] = list[k]; - list.pop(); -} - -function arrayClone(arr, i) { - var copy = new Array(i); - while (i--) - copy[i] = arr[i]; - return copy; -} - -function unwrapListeners(arr) { - var ret = new Array(arr.length); - for (var i = 0; i < ret.length; ++i) { - ret[i] = arr[i].listener || arr[i]; - } - return ret; -} - -// shim for using process in browser -// based off https://github.com/defunctzombie/node-process/blob/master/browser.js - -function defaultSetTimout() { - throw new Error('setTimeout has not been defined'); -} -function defaultClearTimeout () { - throw new Error('clearTimeout has not been defined'); -} -var cachedSetTimeout = defaultSetTimout; -var cachedClearTimeout = defaultClearTimeout; -if (typeof global$1.setTimeout === 'function') { - cachedSetTimeout = setTimeout; -} -if (typeof global$1.clearTimeout === 'function') { - cachedClearTimeout = clearTimeout; -} - -function runTimeout(fun) { - if (cachedSetTimeout === setTimeout) { - //normal enviroments in sane situations - return setTimeout(fun, 0); - } - // if setTimeout wasn't available but was latter defined - if ((cachedSetTimeout === defaultSetTimout || !cachedSetTimeout) && setTimeout) { - cachedSetTimeout = setTimeout; - return setTimeout(fun, 0); - } - try { - // when when somebody has screwed with setTimeout but no I.E. maddness - return cachedSetTimeout(fun, 0); - } catch(e){ - try { - // When we are in I.E. but the script has been evaled so I.E. doesn't trust the global object when called normally - return cachedSetTimeout.call(null, fun, 0); - } catch(e){ - // same as above but when it's a version of I.E. that must have the global object for 'this', hopfully our context correct otherwise it will throw a global error - return cachedSetTimeout.call(this, fun, 0); - } - } - - -} -function runClearTimeout(marker) { - if (cachedClearTimeout === clearTimeout) { - //normal enviroments in sane situations - return clearTimeout(marker); - } - // if clearTimeout wasn't available but was latter defined - if ((cachedClearTimeout === defaultClearTimeout || !cachedClearTimeout) && clearTimeout) { - cachedClearTimeout = clearTimeout; - return clearTimeout(marker); - } - try { - // when when somebody has screwed with setTimeout but no I.E. maddness - return cachedClearTimeout(marker); - } catch (e){ - try { - // When we are in I.E. but the script has been evaled so I.E. doesn't trust the global object when called normally - return cachedClearTimeout.call(null, marker); - } catch (e){ - // same as above but when it's a version of I.E. that must have the global object for 'this', hopfully our context correct otherwise it will throw a global error. - // Some versions of I.E. have different rules for clearTimeout vs setTimeout - return cachedClearTimeout.call(this, marker); - } - } - - - -} -var queue = []; -var draining = false; -var currentQueue; -var queueIndex = -1; - -function cleanUpNextTick() { - if (!draining || !currentQueue) { - return; - } - draining = false; - if (currentQueue.length) { - queue = currentQueue.concat(queue); - } else { - queueIndex = -1; - } - if (queue.length) { - drainQueue(); - } -} - -function drainQueue() { - if (draining) { - return; - } - var timeout = runTimeout(cleanUpNextTick); - draining = true; - - var len = queue.length; - while(len) { - currentQueue = queue; - queue = []; - while (++queueIndex < len) { - if (currentQueue) { - currentQueue[queueIndex].run(); - } - } - queueIndex = -1; - len = queue.length; - } - currentQueue = null; - draining = false; - runClearTimeout(timeout); -} -function nextTick(fun) { - var args = new Array(arguments.length - 1); - if (arguments.length > 1) { - for (var i = 1; i < arguments.length; i++) { - args[i - 1] = arguments[i]; - } - } - queue.push(new Item(fun, args)); - if (queue.length === 1 && !draining) { - runTimeout(drainQueue); - } -} -// v8 likes predictible objects -function Item(fun, array) { - this.fun = fun; - this.array = array; -} -Item.prototype.run = function () { - this.fun.apply(null, this.array); -}; - -// from https://github.com/kumavis/browser-process-hrtime/blob/master/index.js -var performance = global$1.performance || {}; -performance.now || - performance.mozNow || - performance.msNow || - performance.oNow || - performance.webkitNow || - function(){ return (new Date()).getTime() }; - -var inherits; -if (typeof Object.create === 'function'){ - inherits = function inherits(ctor, superCtor) { - // implementation from standard node.js 'util' module - ctor.super_ = superCtor; - ctor.prototype = Object.create(superCtor.prototype, { - constructor: { - value: ctor, - enumerable: false, - writable: true, - configurable: true - } - }); - }; -} else { - inherits = function inherits(ctor, superCtor) { - ctor.super_ = superCtor; - var TempCtor = function () {}; - TempCtor.prototype = superCtor.prototype; - ctor.prototype = new TempCtor(); - ctor.prototype.constructor = ctor; - }; -} -var inherits$1 = inherits; - -var formatRegExp = /%[sdj%]/g; -function format(f) { - if (!isString(f)) { - var objects = []; - for (var i = 0; i < arguments.length; i++) { - objects.push(inspect(arguments[i])); - } - return objects.join(' '); - } - - var i = 1; - var args = arguments; - var len = args.length; - var str = String(f).replace(formatRegExp, function(x) { - if (x === '%%') return '%'; - if (i >= len) return x; - switch (x) { - case '%s': return String(args[i++]); - case '%d': return Number(args[i++]); - case '%j': - try { - return JSON.stringify(args[i++]); - } catch (_) { - return '[Circular]'; - } - default: - return x; - } - }); - for (var x = args[i]; i < len; x = args[++i]) { - if (isNull(x) || !isObject$1(x)) { - str += ' ' + x; - } else { - str += ' ' + inspect(x); - } - } - return str; -} - -// Mark that a method should not be used. -// Returns a modified function which warns once by default. -// If --no-deprecation is set, then it is a no-op. -function deprecate(fn, msg) { - // Allow for deprecating things in the process of starting up. - if (isUndefined(global$1.process)) { - return function() { - return deprecate(fn, msg).apply(this, arguments); - }; - } - - var warned = false; - function deprecated() { - if (!warned) { - { - console.error(msg); - } - warned = true; - } - return fn.apply(this, arguments); - } - - return deprecated; -} - -var debugs = {}; -var debugEnviron; -function debuglog(set) { - if (isUndefined(debugEnviron)) - debugEnviron = ''; - set = set.toUpperCase(); - if (!debugs[set]) { - if (new RegExp('\\b' + set + '\\b', 'i').test(debugEnviron)) { - var pid = 0; - debugs[set] = function() { - var msg = format.apply(null, arguments); - console.error('%s %d: %s', set, pid, msg); - }; - } else { - debugs[set] = function() {}; - } - } - return debugs[set]; -} - -/** - * Echos the value of a value. Trys to print the value out - * in the best way possible given the different types. - * - * @param {Object} obj The object to print out. - * @param {Object} opts Optional options object that alters the output. - */ -/* legacy: obj, showHidden, depth, colors*/ -function inspect(obj, opts) { - // default options - var ctx = { - seen: [], - stylize: stylizeNoColor - }; - // legacy... - if (arguments.length >= 3) ctx.depth = arguments[2]; - if (arguments.length >= 4) ctx.colors = arguments[3]; - if (isBoolean(opts)) { - // legacy... - ctx.showHidden = opts; - } else if (opts) { - // got an "options" object - _extend(ctx, opts); - } - // set default options - if (isUndefined(ctx.showHidden)) ctx.showHidden = false; - if (isUndefined(ctx.depth)) ctx.depth = 2; - if (isUndefined(ctx.colors)) ctx.colors = false; - if (isUndefined(ctx.customInspect)) ctx.customInspect = true; - if (ctx.colors) ctx.stylize = stylizeWithColor; - return formatValue(ctx, obj, ctx.depth); -} - -// http://en.wikipedia.org/wiki/ANSI_escape_code#graphics -inspect.colors = { - 'bold' : [1, 22], - 'italic' : [3, 23], - 'underline' : [4, 24], - 'inverse' : [7, 27], - 'white' : [37, 39], - 'grey' : [90, 39], - 'black' : [30, 39], - 'blue' : [34, 39], - 'cyan' : [36, 39], - 'green' : [32, 39], - 'magenta' : [35, 39], - 'red' : [31, 39], - 'yellow' : [33, 39] -}; - -// Don't use 'blue' not visible on cmd.exe -inspect.styles = { - 'special': 'cyan', - 'number': 'yellow', - 'boolean': 'yellow', - 'undefined': 'grey', - 'null': 'bold', - 'string': 'green', - 'date': 'magenta', - // "name": intentionally not styling - 'regexp': 'red' -}; - - -function stylizeWithColor(str, styleType) { - var style = inspect.styles[styleType]; - - if (style) { - return '\u001b[' + inspect.colors[style][0] + 'm' + str + - '\u001b[' + inspect.colors[style][1] + 'm'; - } else { - return str; - } -} - - -function stylizeNoColor(str, styleType) { - return str; -} - - -function arrayToHash(array) { - var hash = {}; - - array.forEach(function(val, idx) { - hash[val] = true; - }); - - return hash; -} - - -function formatValue(ctx, value, recurseTimes) { - // Provide a hook for user-specified inspect functions. - // Check that value is an object with an inspect function on it - if (ctx.customInspect && - value && - isFunction(value.inspect) && - // Filter out the util module, it's inspect function is special - value.inspect !== inspect && - // Also filter out any prototype objects using the circular check. - !(value.constructor && value.constructor.prototype === value)) { - var ret = value.inspect(recurseTimes, ctx); - if (!isString(ret)) { - ret = formatValue(ctx, ret, recurseTimes); - } - return ret; - } - - // Primitive types cannot have properties - var primitive = formatPrimitive(ctx, value); - if (primitive) { - return primitive; - } - - // Look up the keys of the object. - var keys = Object.keys(value); - var visibleKeys = arrayToHash(keys); - - if (ctx.showHidden) { - keys = Object.getOwnPropertyNames(value); - } - - // IE doesn't make error fields non-enumerable - // http://msdn.microsoft.com/en-us/library/ie/dww52sbt(v=vs.94).aspx - if (isError(value) - && (keys.indexOf('message') >= 0 || keys.indexOf('description') >= 0)) { - return formatError(value); - } - - // Some type of object without properties can be shortcutted. - if (keys.length === 0) { - if (isFunction(value)) { - var name = value.name ? ': ' + value.name : ''; - return ctx.stylize('[Function' + name + ']', 'special'); - } - if (isRegExp(value)) { - return ctx.stylize(RegExp.prototype.toString.call(value), 'regexp'); - } - if (isDate(value)) { - return ctx.stylize(Date.prototype.toString.call(value), 'date'); - } - if (isError(value)) { - return formatError(value); - } - } - - var base = '', array = false, braces = ['{', '}']; - - // Make Array say that they are Array - if (isArray(value)) { - array = true; - braces = ['[', ']']; - } - - // Make functions say that they are functions - if (isFunction(value)) { - var n = value.name ? ': ' + value.name : ''; - base = ' [Function' + n + ']'; - } - - // Make RegExps say that they are RegExps - if (isRegExp(value)) { - base = ' ' + RegExp.prototype.toString.call(value); - } - - // Make dates with properties first say the date - if (isDate(value)) { - base = ' ' + Date.prototype.toUTCString.call(value); - } - - // Make error with message first say the error - if (isError(value)) { - base = ' ' + formatError(value); - } - - if (keys.length === 0 && (!array || value.length == 0)) { - return braces[0] + base + braces[1]; - } - - if (recurseTimes < 0) { - if (isRegExp(value)) { - return ctx.stylize(RegExp.prototype.toString.call(value), 'regexp'); - } else { - return ctx.stylize('[Object]', 'special'); - } - } - - ctx.seen.push(value); - - var output; - if (array) { - output = formatArray(ctx, value, recurseTimes, visibleKeys, keys); - } else { - output = keys.map(function(key) { - return formatProperty(ctx, value, recurseTimes, visibleKeys, key, array); - }); - } - - ctx.seen.pop(); - - return reduceToSingleString(output, base, braces); -} - - -function formatPrimitive(ctx, value) { - if (isUndefined(value)) - return ctx.stylize('undefined', 'undefined'); - if (isString(value)) { - var simple = '\'' + JSON.stringify(value).replace(/^"|"$/g, '') - .replace(/'/g, "\\'") - .replace(/\\"/g, '"') + '\''; - return ctx.stylize(simple, 'string'); - } - if (isNumber(value)) - return ctx.stylize('' + value, 'number'); - if (isBoolean(value)) - return ctx.stylize('' + value, 'boolean'); - // For some reason typeof null is "object", so special case here. - if (isNull(value)) - return ctx.stylize('null', 'null'); -} - - -function formatError(value) { - return '[' + Error.prototype.toString.call(value) + ']'; -} - - -function formatArray(ctx, value, recurseTimes, visibleKeys, keys) { - var output = []; - for (var i = 0, l = value.length; i < l; ++i) { - if (hasOwnProperty(value, String(i))) { - output.push(formatProperty(ctx, value, recurseTimes, visibleKeys, - String(i), true)); - } else { - output.push(''); - } - } - keys.forEach(function(key) { - if (!key.match(/^\d+$/)) { - output.push(formatProperty(ctx, value, recurseTimes, visibleKeys, - key, true)); - } - }); - return output; -} - - -function formatProperty(ctx, value, recurseTimes, visibleKeys, key, array) { - var name, str, desc; - desc = Object.getOwnPropertyDescriptor(value, key) || { value: value[key] }; - if (desc.get) { - if (desc.set) { - str = ctx.stylize('[Getter/Setter]', 'special'); - } else { - str = ctx.stylize('[Getter]', 'special'); - } - } else { - if (desc.set) { - str = ctx.stylize('[Setter]', 'special'); - } - } - if (!hasOwnProperty(visibleKeys, key)) { - name = '[' + key + ']'; - } - if (!str) { - if (ctx.seen.indexOf(desc.value) < 0) { - if (isNull(recurseTimes)) { - str = formatValue(ctx, desc.value, null); - } else { - str = formatValue(ctx, desc.value, recurseTimes - 1); - } - if (str.indexOf('\n') > -1) { - if (array) { - str = str.split('\n').map(function(line) { - return ' ' + line; - }).join('\n').substr(2); - } else { - str = '\n' + str.split('\n').map(function(line) { - return ' ' + line; - }).join('\n'); - } - } - } else { - str = ctx.stylize('[Circular]', 'special'); - } - } - if (isUndefined(name)) { - if (array && key.match(/^\d+$/)) { - return str; - } - name = JSON.stringify('' + key); - if (name.match(/^"([a-zA-Z_][a-zA-Z_0-9]*)"$/)) { - name = name.substr(1, name.length - 2); - name = ctx.stylize(name, 'name'); - } else { - name = name.replace(/'/g, "\\'") - .replace(/\\"/g, '"') - .replace(/(^"|"$)/g, "'"); - name = ctx.stylize(name, 'string'); - } - } - - return name + ': ' + str; -} - - -function reduceToSingleString(output, base, braces) { - var length = output.reduce(function(prev, cur) { - if (cur.indexOf('\n') >= 0) ; - return prev + cur.replace(/\u001b\[\d\d?m/g, '').length + 1; - }, 0); - - if (length > 60) { - return braces[0] + - (base === '' ? '' : base + '\n ') + - ' ' + - output.join(',\n ') + - ' ' + - braces[1]; - } - - return braces[0] + base + ' ' + output.join(', ') + ' ' + braces[1]; -} - - -// NOTE: These type checking functions intentionally don't use `instanceof` -// because it is fragile and can be easily faked with `Object.create()`. -function isArray(ar) { - return Array.isArray(ar); -} - -function isBoolean(arg) { - return typeof arg === 'boolean'; -} - -function isNull(arg) { - return arg === null; -} - -function isNumber(arg) { - return typeof arg === 'number'; -} - -function isString(arg) { - return typeof arg === 'string'; -} - -function isUndefined(arg) { - return arg === void 0; -} - -function isRegExp(re) { - return isObject$1(re) && objectToString(re) === '[object RegExp]'; -} - -function isObject$1(arg) { - return typeof arg === 'object' && arg !== null; -} - -function isDate(d) { - return isObject$1(d) && objectToString(d) === '[object Date]'; -} - -function isError(e) { - return isObject$1(e) && - (objectToString(e) === '[object Error]' || e instanceof Error); -} - -function isFunction(arg) { - return typeof arg === 'function'; -} - -function objectToString(o) { - return Object.prototype.toString.call(o); -} - -function _extend(origin, add) { - // Don't do anything if add isn't an object - if (!add || !isObject$1(add)) return origin; - - var keys = Object.keys(add); - var i = keys.length; - while (i--) { - origin[keys[i]] = add[keys[i]]; - } - return origin; -} -function hasOwnProperty(obj, prop) { - return Object.prototype.hasOwnProperty.call(obj, prop); -} - -function BufferList() { - this.head = null; - this.tail = null; - this.length = 0; -} - -BufferList.prototype.push = function (v) { - var entry = { data: v, next: null }; - if (this.length > 0) this.tail.next = entry;else this.head = entry; - this.tail = entry; - ++this.length; -}; - -BufferList.prototype.unshift = function (v) { - var entry = { data: v, next: this.head }; - if (this.length === 0) this.tail = entry; - this.head = entry; - ++this.length; -}; - -BufferList.prototype.shift = function () { - if (this.length === 0) return; - var ret = this.head.data; - if (this.length === 1) this.head = this.tail = null;else this.head = this.head.next; - --this.length; - return ret; -}; - -BufferList.prototype.clear = function () { - this.head = this.tail = null; - this.length = 0; -}; - -BufferList.prototype.join = function (s) { - if (this.length === 0) return ''; - var p = this.head; - var ret = '' + p.data; - while (p = p.next) { - ret += s + p.data; - }return ret; -}; - -BufferList.prototype.concat = function (n) { - if (this.length === 0) return Buffer.alloc(0); - if (this.length === 1) return this.head.data; - var ret = Buffer.allocUnsafe(n >>> 0); - var p = this.head; - var i = 0; - while (p) { - p.data.copy(ret, i); - i += p.data.length; - p = p.next; - } - return ret; -}; - -// Copyright Joyent, Inc. and other Node contributors. -var isBufferEncoding = Buffer.isEncoding - || function(encoding) { - switch (encoding && encoding.toLowerCase()) { - case 'hex': case 'utf8': case 'utf-8': case 'ascii': case 'binary': case 'base64': case 'ucs2': case 'ucs-2': case 'utf16le': case 'utf-16le': case 'raw': return true; - default: return false; - } - }; - - -function assertEncoding(encoding) { - if (encoding && !isBufferEncoding(encoding)) { - throw new Error('Unknown encoding: ' + encoding); - } -} - -// StringDecoder provides an interface for efficiently splitting a series of -// buffers into a series of JS strings without breaking apart multi-byte -// characters. CESU-8 is handled as part of the UTF-8 encoding. -// -// @TODO Handling all encodings inside a single object makes it very difficult -// to reason about this code, so it should be split up in the future. -// @TODO There should be a utf8-strict encoding that rejects invalid UTF-8 code -// points as used by CESU-8. -function StringDecoder(encoding) { - this.encoding = (encoding || 'utf8').toLowerCase().replace(/[-_]/, ''); - assertEncoding(encoding); - switch (this.encoding) { - case 'utf8': - // CESU-8 represents each of Surrogate Pair by 3-bytes - this.surrogateSize = 3; - break; - case 'ucs2': - case 'utf16le': - // UTF-16 represents each of Surrogate Pair by 2-bytes - this.surrogateSize = 2; - this.detectIncompleteChar = utf16DetectIncompleteChar; - break; - case 'base64': - // Base-64 stores 3 bytes in 4 chars, and pads the remainder. - this.surrogateSize = 3; - this.detectIncompleteChar = base64DetectIncompleteChar; - break; - default: - this.write = passThroughWrite; - return; - } - - // Enough space to store all bytes of a single character. UTF-8 needs 4 - // bytes, but CESU-8 may require up to 6 (3 bytes per surrogate). - this.charBuffer = new Buffer(6); - // Number of bytes received for the current incomplete multi-byte character. - this.charReceived = 0; - // Number of bytes expected for the current incomplete multi-byte character. - this.charLength = 0; -} - -// write decodes the given buffer and returns it as JS string that is -// guaranteed to not contain any partial multi-byte characters. Any partial -// character found at the end of the buffer is buffered up, and will be -// returned when calling write again with the remaining bytes. -// -// Note: Converting a Buffer containing an orphan surrogate to a String -// currently works, but converting a String to a Buffer (via `new Buffer`, or -// Buffer#write) will replace incomplete surrogates with the unicode -// replacement character. See https://codereview.chromium.org/121173009/ . -StringDecoder.prototype.write = function(buffer) { - var charStr = ''; - // if our last write ended with an incomplete multibyte character - while (this.charLength) { - // determine how many remaining bytes this buffer has to offer for this char - var available = (buffer.length >= this.charLength - this.charReceived) ? - this.charLength - this.charReceived : - buffer.length; - - // add the new bytes to the char buffer - buffer.copy(this.charBuffer, this.charReceived, 0, available); - this.charReceived += available; - - if (this.charReceived < this.charLength) { - // still not enough chars in this buffer? wait for more ... - return ''; - } - - // remove bytes belonging to the current character from the buffer - buffer = buffer.slice(available, buffer.length); - - // get the character that was split - charStr = this.charBuffer.slice(0, this.charLength).toString(this.encoding); - - // CESU-8: lead surrogate (D800-DBFF) is also the incomplete character - var charCode = charStr.charCodeAt(charStr.length - 1); - if (charCode >= 0xD800 && charCode <= 0xDBFF) { - this.charLength += this.surrogateSize; - charStr = ''; - continue; - } - this.charReceived = this.charLength = 0; - - // if there are no more bytes in this buffer, just emit our char - if (buffer.length === 0) { - return charStr; - } - break; - } - - // determine and set charLength / charReceived - this.detectIncompleteChar(buffer); - - var end = buffer.length; - if (this.charLength) { - // buffer the incomplete character bytes we got - buffer.copy(this.charBuffer, 0, buffer.length - this.charReceived, end); - end -= this.charReceived; - } - - charStr += buffer.toString(this.encoding, 0, end); - - var end = charStr.length - 1; - var charCode = charStr.charCodeAt(end); - // CESU-8: lead surrogate (D800-DBFF) is also the incomplete character - if (charCode >= 0xD800 && charCode <= 0xDBFF) { - var size = this.surrogateSize; - this.charLength += size; - this.charReceived += size; - this.charBuffer.copy(this.charBuffer, size, 0, size); - buffer.copy(this.charBuffer, 0, 0, size); - return charStr.substring(0, end); - } - - // or just emit the charStr - return charStr; -}; - -// detectIncompleteChar determines if there is an incomplete UTF-8 character at -// the end of the given buffer. If so, it sets this.charLength to the byte -// length that character, and sets this.charReceived to the number of bytes -// that are available for this character. -StringDecoder.prototype.detectIncompleteChar = function(buffer) { - // determine how many bytes we have to check at the end of this buffer - var i = (buffer.length >= 3) ? 3 : buffer.length; - - // Figure out if one of the last i bytes of our buffer announces an - // incomplete char. - for (; i > 0; i--) { - var c = buffer[buffer.length - i]; - - // See http://en.wikipedia.org/wiki/UTF-8#Description - - // 110XXXXX - if (i == 1 && c >> 5 == 0x06) { - this.charLength = 2; - break; - } - - // 1110XXXX - if (i <= 2 && c >> 4 == 0x0E) { - this.charLength = 3; - break; - } - - // 11110XXX - if (i <= 3 && c >> 3 == 0x1E) { - this.charLength = 4; - break; - } - } - this.charReceived = i; -}; - -StringDecoder.prototype.end = function(buffer) { - var res = ''; - if (buffer && buffer.length) - res = this.write(buffer); - - if (this.charReceived) { - var cr = this.charReceived; - var buf = this.charBuffer; - var enc = this.encoding; - res += buf.slice(0, cr).toString(enc); - } - - return res; -}; - -function passThroughWrite(buffer) { - return buffer.toString(this.encoding); -} - -function utf16DetectIncompleteChar(buffer) { - this.charReceived = buffer.length % 2; - this.charLength = this.charReceived ? 2 : 0; -} - -function base64DetectIncompleteChar(buffer) { - this.charReceived = buffer.length % 3; - this.charLength = this.charReceived ? 3 : 0; -} - -Readable.ReadableState = ReadableState; - -var debug = debuglog('stream'); -inherits$1(Readable, EventEmitter); - -function prependListener(emitter, event, fn) { - // Sadly this is not cacheable as some libraries bundle their own - // event emitter implementation with them. - if (typeof emitter.prependListener === 'function') { - return emitter.prependListener(event, fn); - } else { - // This is a hack to make sure that our error handler is attached before any - // userland ones. NEVER DO THIS. This is here only because this code needs - // to continue to work with older versions of Node.js that do not include - // the prependListener() method. The goal is to eventually remove this hack. - if (!emitter._events || !emitter._events[event]) - emitter.on(event, fn); - else if (Array.isArray(emitter._events[event])) - emitter._events[event].unshift(fn); - else - emitter._events[event] = [fn, emitter._events[event]]; - } -} -function listenerCount (emitter, type) { - return emitter.listeners(type).length; -} -function ReadableState(options, stream) { - - options = options || {}; - - // object stream flag. Used to make read(n) ignore n and to - // make all the buffer merging and length checks go away - this.objectMode = !!options.objectMode; - - if (stream instanceof Duplex) this.objectMode = this.objectMode || !!options.readableObjectMode; - - // the point at which it stops calling _read() to fill the buffer - // Note: 0 is a valid value, means "don't call _read preemptively ever" - var hwm = options.highWaterMark; - var defaultHwm = this.objectMode ? 16 : 16 * 1024; - this.highWaterMark = hwm || hwm === 0 ? hwm : defaultHwm; - - // cast to ints. - this.highWaterMark = ~ ~this.highWaterMark; - - // A linked list is used to store data chunks instead of an array because the - // linked list can remove elements from the beginning faster than - // array.shift() - this.buffer = new BufferList(); - this.length = 0; - this.pipes = null; - this.pipesCount = 0; - this.flowing = null; - this.ended = false; - this.endEmitted = false; - this.reading = false; - - // a flag to be able to tell if the onwrite cb is called immediately, - // or on a later tick. We set this to true at first, because any - // actions that shouldn't happen until "later" should generally also - // not happen before the first write call. - this.sync = true; - - // whenever we return null, then we set a flag to say - // that we're awaiting a 'readable' event emission. - this.needReadable = false; - this.emittedReadable = false; - this.readableListening = false; - this.resumeScheduled = false; - - // Crypto is kind of old and crusty. Historically, its default string - // encoding is 'binary' so we have to make this configurable. - // Everything else in the universe uses 'utf8', though. - this.defaultEncoding = options.defaultEncoding || 'utf8'; - - // when piping, we only care about 'readable' events that happen - // after read()ing all the bytes and not getting any pushback. - this.ranOut = false; - - // the number of writers that are awaiting a drain event in .pipe()s - this.awaitDrain = 0; - - // if true, a maybeReadMore has been scheduled - this.readingMore = false; - - this.decoder = null; - this.encoding = null; - if (options.encoding) { - this.decoder = new StringDecoder(options.encoding); - this.encoding = options.encoding; - } -} -function Readable(options) { - - if (!(this instanceof Readable)) return new Readable(options); - - this._readableState = new ReadableState(options, this); - - // legacy - this.readable = true; - - if (options && typeof options.read === 'function') this._read = options.read; - - EventEmitter.call(this); -} - -// Manually shove something into the read() buffer. -// This returns true if the highWaterMark has not been hit yet, -// similar to how Writable.write() returns true if you should -// write() some more. -Readable.prototype.push = function (chunk, encoding) { - var state = this._readableState; - - if (!state.objectMode && typeof chunk === 'string') { - encoding = encoding || state.defaultEncoding; - if (encoding !== state.encoding) { - chunk = Buffer.from(chunk, encoding); - encoding = ''; - } - } - - return readableAddChunk(this, state, chunk, encoding, false); -}; - -// Unshift should *always* be something directly out of read() -Readable.prototype.unshift = function (chunk) { - var state = this._readableState; - return readableAddChunk(this, state, chunk, '', true); -}; - -Readable.prototype.isPaused = function () { - return this._readableState.flowing === false; -}; - -function readableAddChunk(stream, state, chunk, encoding, addToFront) { - var er = chunkInvalid(state, chunk); - if (er) { - stream.emit('error', er); - } else if (chunk === null) { - state.reading = false; - onEofChunk(stream, state); - } else if (state.objectMode || chunk && chunk.length > 0) { - if (state.ended && !addToFront) { - var e = new Error('stream.push() after EOF'); - stream.emit('error', e); - } else if (state.endEmitted && addToFront) { - var _e = new Error('stream.unshift() after end event'); - stream.emit('error', _e); - } else { - var skipAdd; - if (state.decoder && !addToFront && !encoding) { - chunk = state.decoder.write(chunk); - skipAdd = !state.objectMode && chunk.length === 0; - } - - if (!addToFront) state.reading = false; - - // Don't add to the buffer if we've decoded to an empty string chunk and - // we're not in object mode - if (!skipAdd) { - // if we want the data now, just emit it. - if (state.flowing && state.length === 0 && !state.sync) { - stream.emit('data', chunk); - stream.read(0); - } else { - // update the buffer info. - state.length += state.objectMode ? 1 : chunk.length; - if (addToFront) state.buffer.unshift(chunk);else state.buffer.push(chunk); - - if (state.needReadable) emitReadable(stream); - } - } - - maybeReadMore(stream, state); - } - } else if (!addToFront) { - state.reading = false; - } - - return needMoreData(state); -} - -// if it's past the high water mark, we can push in some more. -// Also, if we have no data yet, we can stand some -// more bytes. This is to work around cases where hwm=0, -// such as the repl. Also, if the push() triggered a -// readable event, and the user called read(largeNumber) such that -// needReadable was set, then we ought to push more, so that another -// 'readable' event will be triggered. -function needMoreData(state) { - return !state.ended && (state.needReadable || state.length < state.highWaterMark || state.length === 0); -} - -// backwards compatibility. -Readable.prototype.setEncoding = function (enc) { - this._readableState.decoder = new StringDecoder(enc); - this._readableState.encoding = enc; - return this; -}; - -// Don't raise the hwm > 8MB -var MAX_HWM = 0x800000; -function computeNewHighWaterMark(n) { - if (n >= MAX_HWM) { - n = MAX_HWM; - } else { - // Get the next highest power of 2 to prevent increasing hwm excessively in - // tiny amounts - n--; - n |= n >>> 1; - n |= n >>> 2; - n |= n >>> 4; - n |= n >>> 8; - n |= n >>> 16; - n++; - } - return n; -} - -// This function is designed to be inlinable, so please take care when making -// changes to the function body. -function howMuchToRead(n, state) { - if (n <= 0 || state.length === 0 && state.ended) return 0; - if (state.objectMode) return 1; - if (n !== n) { - // Only flow one buffer at a time - if (state.flowing && state.length) return state.buffer.head.data.length;else return state.length; - } - // If we're asking for more than the current hwm, then raise the hwm. - if (n > state.highWaterMark) state.highWaterMark = computeNewHighWaterMark(n); - if (n <= state.length) return n; - // Don't have enough - if (!state.ended) { - state.needReadable = true; - return 0; - } - return state.length; -} - -// you can override either this method, or the async _read(n) below. -Readable.prototype.read = function (n) { - debug('read', n); - n = parseInt(n, 10); - var state = this._readableState; - var nOrig = n; - - if (n !== 0) state.emittedReadable = false; - - // if we're doing read(0) to trigger a readable event, but we - // already have a bunch of data in the buffer, then just trigger - // the 'readable' event and move on. - if (n === 0 && state.needReadable && (state.length >= state.highWaterMark || state.ended)) { - debug('read: emitReadable', state.length, state.ended); - if (state.length === 0 && state.ended) endReadable(this);else emitReadable(this); - return null; - } - - n = howMuchToRead(n, state); - - // if we've ended, and we're now clear, then finish it up. - if (n === 0 && state.ended) { - if (state.length === 0) endReadable(this); - return null; - } - - // All the actual chunk generation logic needs to be - // *below* the call to _read. The reason is that in certain - // synthetic stream cases, such as passthrough streams, _read - // may be a completely synchronous operation which may change - // the state of the read buffer, providing enough data when - // before there was *not* enough. - // - // So, the steps are: - // 1. Figure out what the state of things will be after we do - // a read from the buffer. - // - // 2. If that resulting state will trigger a _read, then call _read. - // Note that this may be asynchronous, or synchronous. Yes, it is - // deeply ugly to write APIs this way, but that still doesn't mean - // that the Readable class should behave improperly, as streams are - // designed to be sync/async agnostic. - // Take note if the _read call is sync or async (ie, if the read call - // has returned yet), so that we know whether or not it's safe to emit - // 'readable' etc. - // - // 3. Actually pull the requested chunks out of the buffer and return. - - // if we need a readable event, then we need to do some reading. - var doRead = state.needReadable; - debug('need readable', doRead); - - // if we currently have less than the highWaterMark, then also read some - if (state.length === 0 || state.length - n < state.highWaterMark) { - doRead = true; - debug('length less than watermark', doRead); - } - - // however, if we've ended, then there's no point, and if we're already - // reading, then it's unnecessary. - if (state.ended || state.reading) { - doRead = false; - debug('reading or ended', doRead); - } else if (doRead) { - debug('do read'); - state.reading = true; - state.sync = true; - // if the length is currently zero, then we *need* a readable event. - if (state.length === 0) state.needReadable = true; - // call internal read method - this._read(state.highWaterMark); - state.sync = false; - // If _read pushed data synchronously, then `reading` will be false, - // and we need to re-evaluate how much data we can return to the user. - if (!state.reading) n = howMuchToRead(nOrig, state); - } - - var ret; - if (n > 0) ret = fromList(n, state);else ret = null; - - if (ret === null) { - state.needReadable = true; - n = 0; - } else { - state.length -= n; - } - - if (state.length === 0) { - // If we have nothing in the buffer, then we want to know - // as soon as we *do* get something into the buffer. - if (!state.ended) state.needReadable = true; - - // If we tried to read() past the EOF, then emit end on the next tick. - if (nOrig !== n && state.ended) endReadable(this); - } - - if (ret !== null) this.emit('data', ret); - - return ret; -}; - -function chunkInvalid(state, chunk) { - var er = null; - if (!isBuffer(chunk) && typeof chunk !== 'string' && chunk !== null && chunk !== undefined && !state.objectMode) { - er = new TypeError('Invalid non-string/buffer chunk'); - } - return er; -} - -function onEofChunk(stream, state) { - if (state.ended) return; - if (state.decoder) { - var chunk = state.decoder.end(); - if (chunk && chunk.length) { - state.buffer.push(chunk); - state.length += state.objectMode ? 1 : chunk.length; - } - } - state.ended = true; - - // emit 'readable' now to make sure it gets picked up. - emitReadable(stream); -} - -// Don't emit readable right away in sync mode, because this can trigger -// another read() call => stack overflow. This way, it might trigger -// a nextTick recursion warning, but that's not so bad. -function emitReadable(stream) { - var state = stream._readableState; - state.needReadable = false; - if (!state.emittedReadable) { - debug('emitReadable', state.flowing); - state.emittedReadable = true; - if (state.sync) nextTick(emitReadable_, stream);else emitReadable_(stream); - } -} - -function emitReadable_(stream) { - debug('emit readable'); - stream.emit('readable'); - flow(stream); -} - -// at this point, the user has presumably seen the 'readable' event, -// and called read() to consume some data. that may have triggered -// in turn another _read(n) call, in which case reading = true if -// it's in progress. -// However, if we're not ended, or reading, and the length < hwm, -// then go ahead and try to read some more preemptively. -function maybeReadMore(stream, state) { - if (!state.readingMore) { - state.readingMore = true; - nextTick(maybeReadMore_, stream, state); - } -} - -function maybeReadMore_(stream, state) { - var len = state.length; - while (!state.reading && !state.flowing && !state.ended && state.length < state.highWaterMark) { - debug('maybeReadMore read 0'); - stream.read(0); - if (len === state.length) - // didn't get any data, stop spinning. - break;else len = state.length; - } - state.readingMore = false; -} - -// abstract method. to be overridden in specific implementation classes. -// call cb(er, data) where data is <= n in length. -// for virtual (non-string, non-buffer) streams, "length" is somewhat -// arbitrary, and perhaps not very meaningful. -Readable.prototype._read = function (n) { - this.emit('error', new Error('not implemented')); -}; - -Readable.prototype.pipe = function (dest, pipeOpts) { - var src = this; - var state = this._readableState; - - switch (state.pipesCount) { - case 0: - state.pipes = dest; - break; - case 1: - state.pipes = [state.pipes, dest]; - break; - default: - state.pipes.push(dest); - break; - } - state.pipesCount += 1; - debug('pipe count=%d opts=%j', state.pipesCount, pipeOpts); - - var doEnd = (!pipeOpts || pipeOpts.end !== false); - - var endFn = doEnd ? onend : cleanup; - if (state.endEmitted) nextTick(endFn);else src.once('end', endFn); - - dest.on('unpipe', onunpipe); - function onunpipe(readable) { - debug('onunpipe'); - if (readable === src) { - cleanup(); - } - } - - function onend() { - debug('onend'); - dest.end(); - } - - // when the dest drains, it reduces the awaitDrain counter - // on the source. This would be more elegant with a .once() - // handler in flow(), but adding and removing repeatedly is - // too slow. - var ondrain = pipeOnDrain(src); - dest.on('drain', ondrain); - - var cleanedUp = false; - function cleanup() { - debug('cleanup'); - // cleanup event handlers once the pipe is broken - dest.removeListener('close', onclose); - dest.removeListener('finish', onfinish); - dest.removeListener('drain', ondrain); - dest.removeListener('error', onerror); - dest.removeListener('unpipe', onunpipe); - src.removeListener('end', onend); - src.removeListener('end', cleanup); - src.removeListener('data', ondata); - - cleanedUp = true; - - // if the reader is waiting for a drain event from this - // specific writer, then it would cause it to never start - // flowing again. - // So, if this is awaiting a drain, then we just call it now. - // If we don't know, then assume that we are waiting for one. - if (state.awaitDrain && (!dest._writableState || dest._writableState.needDrain)) ondrain(); - } - - // If the user pushes more data while we're writing to dest then we'll end up - // in ondata again. However, we only want to increase awaitDrain once because - // dest will only emit one 'drain' event for the multiple writes. - // => Introduce a guard on increasing awaitDrain. - var increasedAwaitDrain = false; - src.on('data', ondata); - function ondata(chunk) { - debug('ondata'); - increasedAwaitDrain = false; - var ret = dest.write(chunk); - if (false === ret && !increasedAwaitDrain) { - // If the user unpiped during `dest.write()`, it is possible - // to get stuck in a permanently paused state if that write - // also returned false. - // => Check whether `dest` is still a piping destination. - if ((state.pipesCount === 1 && state.pipes === dest || state.pipesCount > 1 && indexOf(state.pipes, dest) !== -1) && !cleanedUp) { - debug('false write response, pause', src._readableState.awaitDrain); - src._readableState.awaitDrain++; - increasedAwaitDrain = true; - } - src.pause(); - } - } - - // if the dest has an error, then stop piping into it. - // however, don't suppress the throwing behavior for this. - function onerror(er) { - debug('onerror', er); - unpipe(); - dest.removeListener('error', onerror); - if (listenerCount(dest, 'error') === 0) dest.emit('error', er); - } - - // Make sure our error handler is attached before userland ones. - prependListener(dest, 'error', onerror); - - // Both close and finish should trigger unpipe, but only once. - function onclose() { - dest.removeListener('finish', onfinish); - unpipe(); - } - dest.once('close', onclose); - function onfinish() { - debug('onfinish'); - dest.removeListener('close', onclose); - unpipe(); - } - dest.once('finish', onfinish); - - function unpipe() { - debug('unpipe'); - src.unpipe(dest); - } - - // tell the dest that it's being piped to - dest.emit('pipe', src); - - // start the flow if it hasn't been started already. - if (!state.flowing) { - debug('pipe resume'); - src.resume(); - } - - return dest; -}; - -function pipeOnDrain(src) { - return function () { - var state = src._readableState; - debug('pipeOnDrain', state.awaitDrain); - if (state.awaitDrain) state.awaitDrain--; - if (state.awaitDrain === 0 && src.listeners('data').length) { - state.flowing = true; - flow(src); - } - }; -} - -Readable.prototype.unpipe = function (dest) { - var state = this._readableState; - - // if we're not piping anywhere, then do nothing. - if (state.pipesCount === 0) return this; - - // just one destination. most common case. - if (state.pipesCount === 1) { - // passed in one, but it's not the right one. - if (dest && dest !== state.pipes) return this; - - if (!dest) dest = state.pipes; - - // got a match. - state.pipes = null; - state.pipesCount = 0; - state.flowing = false; - if (dest) dest.emit('unpipe', this); - return this; - } - - // slow case. multiple pipe destinations. - - if (!dest) { - // remove all. - var dests = state.pipes; - var len = state.pipesCount; - state.pipes = null; - state.pipesCount = 0; - state.flowing = false; - - for (var _i = 0; _i < len; _i++) { - dests[_i].emit('unpipe', this); - }return this; - } - - // try to find the right one. - var i = indexOf(state.pipes, dest); - if (i === -1) return this; - - state.pipes.splice(i, 1); - state.pipesCount -= 1; - if (state.pipesCount === 1) state.pipes = state.pipes[0]; - - dest.emit('unpipe', this); - - return this; -}; - -// set up data events if they are asked for -// Ensure readable listeners eventually get something -Readable.prototype.on = function (ev, fn) { - var res = EventEmitter.prototype.on.call(this, ev, fn); - - if (ev === 'data') { - // Start flowing on next tick if stream isn't explicitly paused - if (this._readableState.flowing !== false) this.resume(); - } else if (ev === 'readable') { - var state = this._readableState; - if (!state.endEmitted && !state.readableListening) { - state.readableListening = state.needReadable = true; - state.emittedReadable = false; - if (!state.reading) { - nextTick(nReadingNextTick, this); - } else if (state.length) { - emitReadable(this); - } - } - } - - return res; -}; -Readable.prototype.addListener = Readable.prototype.on; - -function nReadingNextTick(self) { - debug('readable nexttick read 0'); - self.read(0); -} - -// pause() and resume() are remnants of the legacy readable stream API -// If the user uses them, then switch into old mode. -Readable.prototype.resume = function () { - var state = this._readableState; - if (!state.flowing) { - debug('resume'); - state.flowing = true; - resume(this, state); - } - return this; -}; - -function resume(stream, state) { - if (!state.resumeScheduled) { - state.resumeScheduled = true; - nextTick(resume_, stream, state); - } -} - -function resume_(stream, state) { - if (!state.reading) { - debug('resume read 0'); - stream.read(0); - } - - state.resumeScheduled = false; - state.awaitDrain = 0; - stream.emit('resume'); - flow(stream); - if (state.flowing && !state.reading) stream.read(0); -} - -Readable.prototype.pause = function () { - debug('call pause flowing=%j', this._readableState.flowing); - if (false !== this._readableState.flowing) { - debug('pause'); - this._readableState.flowing = false; - this.emit('pause'); - } - return this; -}; - -function flow(stream) { - var state = stream._readableState; - debug('flow', state.flowing); - while (state.flowing && stream.read() !== null) {} -} - -// wrap an old-style stream as the async data source. -// This is *not* part of the readable stream interface. -// It is an ugly unfortunate mess of history. -Readable.prototype.wrap = function (stream) { - var state = this._readableState; - var paused = false; - - var self = this; - stream.on('end', function () { - debug('wrapped end'); - if (state.decoder && !state.ended) { - var chunk = state.decoder.end(); - if (chunk && chunk.length) self.push(chunk); - } - - self.push(null); - }); - - stream.on('data', function (chunk) { - debug('wrapped data'); - if (state.decoder) chunk = state.decoder.write(chunk); - - // don't skip over falsy values in objectMode - if (state.objectMode && (chunk === null || chunk === undefined)) return;else if (!state.objectMode && (!chunk || !chunk.length)) return; - - var ret = self.push(chunk); - if (!ret) { - paused = true; - stream.pause(); - } - }); - - // proxy all the other methods. - // important when wrapping filters and duplexes. - for (var i in stream) { - if (this[i] === undefined && typeof stream[i] === 'function') { - this[i] = function (method) { - return function () { - return stream[method].apply(stream, arguments); - }; - }(i); - } - } - - // proxy certain important events. - var events = ['error', 'close', 'destroy', 'pause', 'resume']; - forEach(events, function (ev) { - stream.on(ev, self.emit.bind(self, ev)); - }); - - // when we try to consume some more bytes, simply unpause the - // underlying stream. - self._read = function (n) { - debug('wrapped _read', n); - if (paused) { - paused = false; - stream.resume(); - } - }; - - return self; -}; - -// exposed for testing purposes only. -Readable._fromList = fromList; - -// Pluck off n bytes from an array of buffers. -// Length is the combined lengths of all the buffers in the list. -// This function is designed to be inlinable, so please take care when making -// changes to the function body. -function fromList(n, state) { - // nothing buffered - if (state.length === 0) return null; - - var ret; - if (state.objectMode) ret = state.buffer.shift();else if (!n || n >= state.length) { - // read it all, truncate the list - if (state.decoder) ret = state.buffer.join('');else if (state.buffer.length === 1) ret = state.buffer.head.data;else ret = state.buffer.concat(state.length); - state.buffer.clear(); - } else { - // read part of list - ret = fromListPartial(n, state.buffer, state.decoder); - } - - return ret; -} - -// Extracts only enough buffered data to satisfy the amount requested. -// This function is designed to be inlinable, so please take care when making -// changes to the function body. -function fromListPartial(n, list, hasStrings) { - var ret; - if (n < list.head.data.length) { - // slice is the same for buffers and strings - ret = list.head.data.slice(0, n); - list.head.data = list.head.data.slice(n); - } else if (n === list.head.data.length) { - // first chunk is a perfect match - ret = list.shift(); - } else { - // result spans more than one buffer - ret = hasStrings ? copyFromBufferString(n, list) : copyFromBuffer(n, list); - } - return ret; -} - -// Copies a specified amount of characters from the list of buffered data -// chunks. -// This function is designed to be inlinable, so please take care when making -// changes to the function body. -function copyFromBufferString(n, list) { - var p = list.head; - var c = 1; - var ret = p.data; - n -= ret.length; - while (p = p.next) { - var str = p.data; - var nb = n > str.length ? str.length : n; - if (nb === str.length) ret += str;else ret += str.slice(0, n); - n -= nb; - if (n === 0) { - if (nb === str.length) { - ++c; - if (p.next) list.head = p.next;else list.head = list.tail = null; - } else { - list.head = p; - p.data = str.slice(nb); - } - break; - } - ++c; - } - list.length -= c; - return ret; -} - -// Copies a specified amount of bytes from the list of buffered data chunks. -// This function is designed to be inlinable, so please take care when making -// changes to the function body. -function copyFromBuffer(n, list) { - var ret = Buffer.allocUnsafe(n); - var p = list.head; - var c = 1; - p.data.copy(ret); - n -= p.data.length; - while (p = p.next) { - var buf = p.data; - var nb = n > buf.length ? buf.length : n; - buf.copy(ret, ret.length - n, 0, nb); - n -= nb; - if (n === 0) { - if (nb === buf.length) { - ++c; - if (p.next) list.head = p.next;else list.head = list.tail = null; - } else { - list.head = p; - p.data = buf.slice(nb); - } - break; - } - ++c; - } - list.length -= c; - return ret; -} - -function endReadable(stream) { - var state = stream._readableState; - - // If we get here before consuming all the bytes, then that is a - // bug in node. Should never happen. - if (state.length > 0) throw new Error('"endReadable()" called on non-empty stream'); - - if (!state.endEmitted) { - state.ended = true; - nextTick(endReadableNT, state, stream); - } -} - -function endReadableNT(state, stream) { - // Check that we didn't get one last unshift. - if (!state.endEmitted && state.length === 0) { - state.endEmitted = true; - stream.readable = false; - stream.emit('end'); - } -} - -function forEach(xs, f) { - for (var i = 0, l = xs.length; i < l; i++) { - f(xs[i], i); - } -} - -function indexOf(xs, x) { - for (var i = 0, l = xs.length; i < l; i++) { - if (xs[i] === x) return i; - } - return -1; -} - -// A bit simpler than readable streams. -Writable.WritableState = WritableState; -inherits$1(Writable, EventEmitter); - -function nop() {} - -function WriteReq(chunk, encoding, cb) { - this.chunk = chunk; - this.encoding = encoding; - this.callback = cb; - this.next = null; -} - -function WritableState(options, stream) { - Object.defineProperty(this, 'buffer', { - get: deprecate(function () { - return this.getBuffer(); - }, '_writableState.buffer is deprecated. Use _writableState.getBuffer ' + 'instead.') - }); - options = options || {}; - - // object stream flag to indicate whether or not this stream - // contains buffers or objects. - this.objectMode = !!options.objectMode; - - if (stream instanceof Duplex) this.objectMode = this.objectMode || !!options.writableObjectMode; - - // the point at which write() starts returning false - // Note: 0 is a valid value, means that we always return false if - // the entire buffer is not flushed immediately on write() - var hwm = options.highWaterMark; - var defaultHwm = this.objectMode ? 16 : 16 * 1024; - this.highWaterMark = hwm || hwm === 0 ? hwm : defaultHwm; - - // cast to ints. - this.highWaterMark = ~ ~this.highWaterMark; - - this.needDrain = false; - // at the start of calling end() - this.ending = false; - // when end() has been called, and returned - this.ended = false; - // when 'finish' is emitted - this.finished = false; - - // should we decode strings into buffers before passing to _write? - // this is here so that some node-core streams can optimize string - // handling at a lower level. - var noDecode = options.decodeStrings === false; - this.decodeStrings = !noDecode; - - // Crypto is kind of old and crusty. Historically, its default string - // encoding is 'binary' so we have to make this configurable. - // Everything else in the universe uses 'utf8', though. - this.defaultEncoding = options.defaultEncoding || 'utf8'; - - // not an actual buffer we keep track of, but a measurement - // of how much we're waiting to get pushed to some underlying - // socket or file. - this.length = 0; - - // a flag to see when we're in the middle of a write. - this.writing = false; - - // when true all writes will be buffered until .uncork() call - this.corked = 0; - - // a flag to be able to tell if the onwrite cb is called immediately, - // or on a later tick. We set this to true at first, because any - // actions that shouldn't happen until "later" should generally also - // not happen before the first write call. - this.sync = true; - - // a flag to know if we're processing previously buffered items, which - // may call the _write() callback in the same tick, so that we don't - // end up in an overlapped onwrite situation. - this.bufferProcessing = false; - - // the callback that's passed to _write(chunk,cb) - this.onwrite = function (er) { - onwrite(stream, er); - }; - - // the callback that the user supplies to write(chunk,encoding,cb) - this.writecb = null; - - // the amount that is being written when _write is called. - this.writelen = 0; - - this.bufferedRequest = null; - this.lastBufferedRequest = null; - - // number of pending user-supplied write callbacks - // this must be 0 before 'finish' can be emitted - this.pendingcb = 0; - - // emit prefinish if the only thing we're waiting for is _write cbs - // This is relevant for synchronous Transform streams - this.prefinished = false; - - // True if the error was already emitted and should not be thrown again - this.errorEmitted = false; - - // count buffered requests - this.bufferedRequestCount = 0; - - // allocate the first CorkedRequest, there is always - // one allocated and free to use, and we maintain at most two - this.corkedRequestsFree = new CorkedRequest(this); -} - -WritableState.prototype.getBuffer = function writableStateGetBuffer() { - var current = this.bufferedRequest; - var out = []; - while (current) { - out.push(current); - current = current.next; - } - return out; -}; -function Writable(options) { - - // Writable ctor is applied to Duplexes, though they're not - // instanceof Writable, they're instanceof Readable. - if (!(this instanceof Writable) && !(this instanceof Duplex)) return new Writable(options); - - this._writableState = new WritableState(options, this); - - // legacy. - this.writable = true; - - if (options) { - if (typeof options.write === 'function') this._write = options.write; - - if (typeof options.writev === 'function') this._writev = options.writev; - } - - EventEmitter.call(this); -} - -// Otherwise people can pipe Writable streams, which is just wrong. -Writable.prototype.pipe = function () { - this.emit('error', new Error('Cannot pipe, not readable')); -}; - -function writeAfterEnd(stream, cb) { - var er = new Error('write after end'); - // TODO: defer error events consistently everywhere, not just the cb - stream.emit('error', er); - nextTick(cb, er); -} - -// If we get something that is not a buffer, string, null, or undefined, -// and we're not in objectMode, then that's an error. -// Otherwise stream chunks are all considered to be of length=1, and the -// watermarks determine how many objects to keep in the buffer, rather than -// how many bytes or characters. -function validChunk(stream, state, chunk, cb) { - var valid = true; - var er = false; - // Always throw error if a null is written - // if we are not in object mode then throw - // if it is not a buffer, string, or undefined. - if (chunk === null) { - er = new TypeError('May not write null values to stream'); - } else if (!Buffer.isBuffer(chunk) && typeof chunk !== 'string' && chunk !== undefined && !state.objectMode) { - er = new TypeError('Invalid non-string/buffer chunk'); - } - if (er) { - stream.emit('error', er); - nextTick(cb, er); - valid = false; - } - return valid; -} - -Writable.prototype.write = function (chunk, encoding, cb) { - var state = this._writableState; - var ret = false; - - if (typeof encoding === 'function') { - cb = encoding; - encoding = null; - } - - if (Buffer.isBuffer(chunk)) encoding = 'buffer';else if (!encoding) encoding = state.defaultEncoding; - - if (typeof cb !== 'function') cb = nop; - - if (state.ended) writeAfterEnd(this, cb);else if (validChunk(this, state, chunk, cb)) { - state.pendingcb++; - ret = writeOrBuffer(this, state, chunk, encoding, cb); - } - - return ret; -}; - -Writable.prototype.cork = function () { - var state = this._writableState; - - state.corked++; -}; - -Writable.prototype.uncork = function () { - var state = this._writableState; - - if (state.corked) { - state.corked--; - - if (!state.writing && !state.corked && !state.finished && !state.bufferProcessing && state.bufferedRequest) clearBuffer(this, state); - } -}; - -Writable.prototype.setDefaultEncoding = function setDefaultEncoding(encoding) { - // node::ParseEncoding() requires lower case. - if (typeof encoding === 'string') encoding = encoding.toLowerCase(); - if (!(['hex', 'utf8', 'utf-8', 'ascii', 'binary', 'base64', 'ucs2', 'ucs-2', 'utf16le', 'utf-16le', 'raw'].indexOf((encoding + '').toLowerCase()) > -1)) throw new TypeError('Unknown encoding: ' + encoding); - this._writableState.defaultEncoding = encoding; - return this; -}; - -function decodeChunk(state, chunk, encoding) { - if (!state.objectMode && state.decodeStrings !== false && typeof chunk === 'string') { - chunk = Buffer.from(chunk, encoding); - } - return chunk; -} - -// if we're already writing something, then just put this -// in the queue, and wait our turn. Otherwise, call _write -// If we return false, then we need a drain event, so set that flag. -function writeOrBuffer(stream, state, chunk, encoding, cb) { - chunk = decodeChunk(state, chunk, encoding); - - if (Buffer.isBuffer(chunk)) encoding = 'buffer'; - var len = state.objectMode ? 1 : chunk.length; - - state.length += len; - - var ret = state.length < state.highWaterMark; - // we must ensure that previous needDrain will not be reset to false. - if (!ret) state.needDrain = true; - - if (state.writing || state.corked) { - var last = state.lastBufferedRequest; - state.lastBufferedRequest = new WriteReq(chunk, encoding, cb); - if (last) { - last.next = state.lastBufferedRequest; - } else { - state.bufferedRequest = state.lastBufferedRequest; - } - state.bufferedRequestCount += 1; - } else { - doWrite(stream, state, false, len, chunk, encoding, cb); - } - - return ret; -} - -function doWrite(stream, state, writev, len, chunk, encoding, cb) { - state.writelen = len; - state.writecb = cb; - state.writing = true; - state.sync = true; - if (writev) stream._writev(chunk, state.onwrite);else stream._write(chunk, encoding, state.onwrite); - state.sync = false; -} - -function onwriteError(stream, state, sync, er, cb) { - --state.pendingcb; - if (sync) nextTick(cb, er);else cb(er); - - stream._writableState.errorEmitted = true; - stream.emit('error', er); -} - -function onwriteStateUpdate(state) { - state.writing = false; - state.writecb = null; - state.length -= state.writelen; - state.writelen = 0; -} - -function onwrite(stream, er) { - var state = stream._writableState; - var sync = state.sync; - var cb = state.writecb; - - onwriteStateUpdate(state); - - if (er) onwriteError(stream, state, sync, er, cb);else { - // Check if we're actually ready to finish, but don't emit yet - var finished = needFinish(state); - - if (!finished && !state.corked && !state.bufferProcessing && state.bufferedRequest) { - clearBuffer(stream, state); - } - - if (sync) { - /**/ - nextTick(afterWrite, stream, state, finished, cb); - /**/ - } else { - afterWrite(stream, state, finished, cb); - } - } -} - -function afterWrite(stream, state, finished, cb) { - if (!finished) onwriteDrain(stream, state); - state.pendingcb--; - cb(); - finishMaybe(stream, state); -} - -// Must force callback to be called on nextTick, so that we don't -// emit 'drain' before the write() consumer gets the 'false' return -// value, and has a chance to attach a 'drain' listener. -function onwriteDrain(stream, state) { - if (state.length === 0 && state.needDrain) { - state.needDrain = false; - stream.emit('drain'); - } -} - -// if there's something in the buffer waiting, then process it -function clearBuffer(stream, state) { - state.bufferProcessing = true; - var entry = state.bufferedRequest; - - if (stream._writev && entry && entry.next) { - // Fast case, write everything using _writev() - var l = state.bufferedRequestCount; - var buffer = new Array(l); - var holder = state.corkedRequestsFree; - holder.entry = entry; - - var count = 0; - while (entry) { - buffer[count] = entry; - entry = entry.next; - count += 1; - } - - doWrite(stream, state, true, state.length, buffer, '', holder.finish); - - // doWrite is almost always async, defer these to save a bit of time - // as the hot path ends with doWrite - state.pendingcb++; - state.lastBufferedRequest = null; - if (holder.next) { - state.corkedRequestsFree = holder.next; - holder.next = null; - } else { - state.corkedRequestsFree = new CorkedRequest(state); - } - } else { - // Slow case, write chunks one-by-one - while (entry) { - var chunk = entry.chunk; - var encoding = entry.encoding; - var cb = entry.callback; - var len = state.objectMode ? 1 : chunk.length; - - doWrite(stream, state, false, len, chunk, encoding, cb); - entry = entry.next; - // if we didn't call the onwrite immediately, then - // it means that we need to wait until it does. - // also, that means that the chunk and cb are currently - // being processed, so move the buffer counter past them. - if (state.writing) { - break; - } - } - - if (entry === null) state.lastBufferedRequest = null; - } - - state.bufferedRequestCount = 0; - state.bufferedRequest = entry; - state.bufferProcessing = false; -} - -Writable.prototype._write = function (chunk, encoding, cb) { - cb(new Error('not implemented')); -}; - -Writable.prototype._writev = null; - -Writable.prototype.end = function (chunk, encoding, cb) { - var state = this._writableState; - - if (typeof chunk === 'function') { - cb = chunk; - chunk = null; - encoding = null; - } else if (typeof encoding === 'function') { - cb = encoding; - encoding = null; - } - - if (chunk !== null && chunk !== undefined) this.write(chunk, encoding); - - // .end() fully uncorks - if (state.corked) { - state.corked = 1; - this.uncork(); - } - - // ignore unnecessary end() calls. - if (!state.ending && !state.finished) endWritable(this, state, cb); -}; - -function needFinish(state) { - return state.ending && state.length === 0 && state.bufferedRequest === null && !state.finished && !state.writing; -} - -function prefinish(stream, state) { - if (!state.prefinished) { - state.prefinished = true; - stream.emit('prefinish'); - } -} - -function finishMaybe(stream, state) { - var need = needFinish(state); - if (need) { - if (state.pendingcb === 0) { - prefinish(stream, state); - state.finished = true; - stream.emit('finish'); - } else { - prefinish(stream, state); - } - } - return need; -} - -function endWritable(stream, state, cb) { - state.ending = true; - finishMaybe(stream, state); - if (cb) { - if (state.finished) nextTick(cb);else stream.once('finish', cb); - } - state.ended = true; - stream.writable = false; -} - -// It seems a linked list but it is not -// there will be only 2 of these for each stream -function CorkedRequest(state) { - var _this = this; - - this.next = null; - this.entry = null; - - this.finish = function (err) { - var entry = _this.entry; - _this.entry = null; - while (entry) { - var cb = entry.callback; - state.pendingcb--; - cb(err); - entry = entry.next; - } - if (state.corkedRequestsFree) { - state.corkedRequestsFree.next = _this; - } else { - state.corkedRequestsFree = _this; - } - }; -} - -inherits$1(Duplex, Readable); - -var keys = Object.keys(Writable.prototype); -for (var v = 0; v < keys.length; v++) { - var method = keys[v]; - if (!Duplex.prototype[method]) Duplex.prototype[method] = Writable.prototype[method]; -} -function Duplex(options) { - if (!(this instanceof Duplex)) return new Duplex(options); - - Readable.call(this, options); - Writable.call(this, options); - - if (options && options.readable === false) this.readable = false; - - if (options && options.writable === false) this.writable = false; - - this.allowHalfOpen = true; - if (options && options.allowHalfOpen === false) this.allowHalfOpen = false; - - this.once('end', onend); -} - -// the no-half-open enforcer -function onend() { - // if we allow half-open state, or if the writable side ended, - // then we're ok. - if (this.allowHalfOpen || this._writableState.ended) return; - - // no more data can be written. - // But allow more writes to happen in this tick. - nextTick(onEndNT, this); -} - -function onEndNT(self) { - self.end(); -} - -// a transform stream is a readable/writable stream where you do -inherits$1(Transform, Duplex); - -function TransformState(stream) { - this.afterTransform = function (er, data) { - return afterTransform(stream, er, data); - }; - - this.needTransform = false; - this.transforming = false; - this.writecb = null; - this.writechunk = null; - this.writeencoding = null; -} - -function afterTransform(stream, er, data) { - var ts = stream._transformState; - ts.transforming = false; - - var cb = ts.writecb; - - if (!cb) return stream.emit('error', new Error('no writecb in Transform class')); - - ts.writechunk = null; - ts.writecb = null; - - if (data !== null && data !== undefined) stream.push(data); - - cb(er); - - var rs = stream._readableState; - rs.reading = false; - if (rs.needReadable || rs.length < rs.highWaterMark) { - stream._read(rs.highWaterMark); - } -} -function Transform(options) { - if (!(this instanceof Transform)) return new Transform(options); - - Duplex.call(this, options); - - this._transformState = new TransformState(this); - - // when the writable side finishes, then flush out anything remaining. - var stream = this; - - // start out asking for a readable event once data is transformed. - this._readableState.needReadable = true; - - // we have implemented the _read method, and done the other things - // that Readable wants before the first _read call, so unset the - // sync guard flag. - this._readableState.sync = false; - - if (options) { - if (typeof options.transform === 'function') this._transform = options.transform; - - if (typeof options.flush === 'function') this._flush = options.flush; - } - - this.once('prefinish', function () { - if (typeof this._flush === 'function') this._flush(function (er) { - done(stream, er); - });else done(stream); - }); -} - -Transform.prototype.push = function (chunk, encoding) { - this._transformState.needTransform = false; - return Duplex.prototype.push.call(this, chunk, encoding); -}; - -// This is the part where you do stuff! -// override this function in implementation classes. -// 'chunk' is an input chunk. -// -// Call `push(newChunk)` to pass along transformed output -// to the readable side. You may call 'push' zero or more times. -// -// Call `cb(err)` when you are done with this chunk. If you pass -// an error, then that'll put the hurt on the whole operation. If you -// never call cb(), then you'll never get another chunk. -Transform.prototype._transform = function (chunk, encoding, cb) { - throw new Error('Not implemented'); -}; - -Transform.prototype._write = function (chunk, encoding, cb) { - var ts = this._transformState; - ts.writecb = cb; - ts.writechunk = chunk; - ts.writeencoding = encoding; - if (!ts.transforming) { - var rs = this._readableState; - if (ts.needTransform || rs.needReadable || rs.length < rs.highWaterMark) this._read(rs.highWaterMark); - } -}; - -// Doesn't matter what the args are here. -// _transform does all the work. -// That we got here means that the readable side wants more data. -Transform.prototype._read = function (n) { - var ts = this._transformState; - - if (ts.writechunk !== null && ts.writecb && !ts.transforming) { - ts.transforming = true; - this._transform(ts.writechunk, ts.writeencoding, ts.afterTransform); - } else { - // mark that we need a transform, so that any data that comes in - // will get processed, now that we've asked for it. - ts.needTransform = true; +class CsvError extends Error { + constructor(code, message, options, ...contexts) { + if(Array.isArray(message)) message = message.join(' '); + super(message); + if(Error.captureStackTrace !== undefined){ + Error.captureStackTrace(this, CsvError); + } + this.code = code; + for(const context of contexts){ + for(const key in context){ + const value = context[key]; + this[key] = isBuffer(value) ? value.toString(options.encoding) : value == null ? value : JSON.parse(JSON.stringify(value)); + } + } } -}; - -function done(stream, er) { - if (er) return stream.emit('error', er); - - // if there's nothing in the write buffer, then that means - // that nothing more will ever be provided - var ws = stream._writableState; - var ts = stream._transformState; - - if (ws.length) throw new Error('Calling transform done when ws.length != 0'); - - if (ts.transforming) throw new Error('Calling transform done when still transforming'); - - return stream.push(null); } -inherits$1(PassThrough, Transform); -function PassThrough(options) { - if (!(this instanceof PassThrough)) return new PassThrough(options); - - Transform.call(this, options); -} - -PassThrough.prototype._transform = function (chunk, encoding, cb) { - cb(null, chunk); +const is_object = function(obj){ + return (typeof obj === 'object' && obj !== null && !Array.isArray(obj)); }; -inherits$1(Stream, EventEmitter); -Stream.Readable = Readable; -Stream.Writable = Writable; -Stream.Duplex = Duplex; -Stream.Transform = Transform; -Stream.PassThrough = PassThrough; - -// Backwards-compat with node 0.4.x -Stream.Stream = Stream; - -// old-style streams. Note that the pipe method (the only relevant -// part of this class) is overridden in the Readable class. - -function Stream() { - EventEmitter.call(this); -} - -Stream.prototype.pipe = function(dest, options) { - var source = this; - - function ondata(chunk) { - if (dest.writable) { - if (false === dest.write(chunk) && source.pause) { - source.pause(); +const normalize_columns_array = function(columns){ + const normalizedColumns = []; + for(let i = 0, l = columns.length; i < l; i++){ + const column = columns[i]; + if(column === undefined || column === null || column === false){ + normalizedColumns[i] = { disabled: true }; + }else if(typeof column === 'string'){ + normalizedColumns[i] = { name: column }; + }else if(is_object(column)){ + if(typeof column.name !== 'string'){ + throw new CsvError('CSV_OPTION_COLUMNS_MISSING_NAME', [ + 'Option columns missing name:', + `property "name" is required at position ${i}`, + 'when column is an object literal' + ]); } + normalizedColumns[i] = column; + }else { + throw new CsvError('CSV_INVALID_COLUMN_DEFINITION', [ + 'Invalid column definition:', + 'expect a string or a literal object,', + `got ${JSON.stringify(column)} at position ${i}` + ]); } } - - source.on('data', ondata); - - function ondrain() { - if (source.readable && source.resume) { - source.resume(); - } - } - - dest.on('drain', ondrain); - - // If the 'end' option is not supplied, dest.end() will be called when - // source gets the 'end' or 'close' events. Only dest.end() once. - if (!dest._isStdio && (!options || options.end !== false)) { - source.on('end', onend); - source.on('close', onclose); - } - - var didOnEnd = false; - function onend() { - if (didOnEnd) return; - didOnEnd = true; - - dest.end(); - } - - - function onclose() { - if (didOnEnd) return; - didOnEnd = true; - - if (typeof dest.destroy === 'function') dest.destroy(); - } - - // don't leave dangling pipes when there are errors. - function onerror(er) { - cleanup(); - if (EventEmitter.listenerCount(this, 'error') === 0) { - throw er; // Unhandled stream error in pipe. - } - } - - source.on('error', onerror); - dest.on('error', onerror); - - // remove all the event listeners that were added. - function cleanup() { - source.removeListener('data', ondata); - dest.removeListener('drain', ondrain); - - source.removeListener('end', onend); - source.removeListener('close', onclose); - - source.removeListener('error', onerror); - dest.removeListener('error', onerror); - - source.removeListener('end', cleanup); - source.removeListener('close', cleanup); - - dest.removeListener('close', cleanup); - } - - source.on('end', cleanup); - source.on('close', cleanup); - - dest.on('close', cleanup); - - dest.emit('pipe', source); - - // Allow for unix-like usage: A.pipe(B).pipe(C) - return dest; + return normalizedColumns; }; class ResizeableBuffer{ @@ -5028,1232 +2074,1171 @@ class ResizeableBuffer{ } } -// white space characters -// https://en.wikipedia.org/wiki/Whitespace_character -// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions/Character_Classes#Types -// \f\n\r\t\v\u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff -const tab = 9; -const nl = 10; // \n, 0x0A in hexadecimal, 10 in decimal -const np = 12; -const cr = 13; // \r, 0x0D in hexadécimal, 13 in decimal -const space = 32; -const boms = { - // Note, the following are equals: - // Buffer.from("\ufeff") - // Buffer.from([239, 187, 191]) - // Buffer.from('EFBBBF', 'hex') - 'utf8': Buffer.from([239, 187, 191]), - // Note, the following are equals: - // Buffer.from "\ufeff", 'utf16le - // Buffer.from([255, 254]) - 'utf16le': Buffer.from([255, 254]) +const init_state = function(options){ + return { + bomSkipped: false, + bufBytesStart: 0, + castField: options.cast_function, + commenting: false, + // Current error encountered by a record + error: undefined, + enabled: options.from_line === 1, + escaping: false, + escapeIsQuote: isBuffer(options.escape) && isBuffer(options.quote) && Buffer.compare(options.escape, options.quote) === 0, + // columns can be `false`, `true`, `Array` + expectedRecordLength: Array.isArray(options.columns) ? options.columns.length : undefined, + field: new ResizeableBuffer(20), + firstLineToHeaders: options.cast_first_line_to_header, + needMoreDataSize: Math.max( + // Skip if the remaining buffer smaller than comment + options.comment !== null ? options.comment.length : 0, + // Skip if the remaining buffer can be delimiter + ...options.delimiter.map((delimiter) => delimiter.length), + // Skip if the remaining buffer can be escape sequence + options.quote !== null ? options.quote.length : 0, + ), + previousBuf: undefined, + quoting: false, + stop: false, + rawBuffer: new ResizeableBuffer(100), + record: [], + recordHasError: false, + record_length: 0, + recordDelimiterMaxLength: options.record_delimiter.length === 0 ? 2 : Math.max(...options.record_delimiter.map((v) => v.length)), + trimChars: [Buffer.from(' ', options.encoding)[0], Buffer.from('\t', options.encoding)[0]], + wasQuoting: false, + wasRowDelimiter: false + }; }; -class CsvError extends Error { - constructor(code, message, options, ...contexts) { - if(Array.isArray(message)) message = message.join(' '); - super(message); - if(Error.captureStackTrace !== undefined){ - Error.captureStackTrace(this, CsvError); - } - this.code = code; - for(const context of contexts){ - for(const key in context){ - const value = context[key]; - this[key] = isBuffer(value) ? value.toString(options.encoding) : value == null ? value : JSON.parse(JSON.stringify(value)); - } - } - } -} - const underscore = function(str){ return str.replace(/([A-Z])/g, function(_, match){ return '_' + match.toLowerCase(); }); }; -const isObject = function(obj){ - return (typeof obj === 'object' && obj !== null && !Array.isArray(obj)); -}; - -const isRecordEmpty = function(record){ - return record.every((field) => field == null || field.toString && field.toString().trim() === ''); -}; - -const normalizeColumnsArray = function(columns){ - const normalizedColumns = []; - for(let i = 0, l = columns.length; i < l; i++){ - const column = columns[i]; - if(column === undefined || column === null || column === false){ - normalizedColumns[i] = { disabled: true }; - }else if(typeof column === 'string'){ - normalizedColumns[i] = { name: column }; - }else if(isObject(column)){ - if(typeof column.name !== 'string'){ - throw new CsvError('CSV_OPTION_COLUMNS_MISSING_NAME', [ - 'Option columns missing name:', - `property "name" is required at position ${i}`, - 'when column is an object literal' - ]); - } - normalizedColumns[i] = column; - }else { - throw new CsvError('CSV_INVALID_COLUMN_DEFINITION', [ - 'Invalid column definition:', - 'expect a string or a literal object,', - `got ${JSON.stringify(column)} at position ${i}` - ]); - } - } - return normalizedColumns; -}; - -class Parser extends Transform { - constructor(opts = {}){ - super({...{readableObjectMode: true}, ...opts, encoding: null}); - this.__originalOptions = opts; - this.__normalizeOptions(opts); - } - __normalizeOptions(opts){ - const options = {}; - // Merge with user options - for(const opt in opts){ - options[underscore(opt)] = opts[opt]; - } - // Normalize option `encoding` - // Note: defined first because other options depends on it - // to convert chars/strings into buffers. - if(options.encoding === undefined || options.encoding === true){ - options.encoding = 'utf8'; - }else if(options.encoding === null || options.encoding === false){ - options.encoding = null; - }else if(typeof options.encoding !== 'string' && options.encoding !== null){ - throw new CsvError('CSV_INVALID_OPTION_ENCODING', [ - 'Invalid option encoding:', - 'encoding must be a string or null to return a buffer,', - `got ${JSON.stringify(options.encoding)}` - ], options); - } - // Normalize option `bom` - if(options.bom === undefined || options.bom === null || options.bom === false){ - options.bom = false; - }else if(options.bom !== true){ - throw new CsvError('CSV_INVALID_OPTION_BOM', [ - 'Invalid option bom:', 'bom must be true,', - `got ${JSON.stringify(options.bom)}` - ], options); - } - // Normalize option `cast` - let fnCastField = null; - if(options.cast === undefined || options.cast === null || options.cast === false || options.cast === ''){ - options.cast = undefined; - }else if(typeof options.cast === 'function'){ - fnCastField = options.cast; - options.cast = true; - }else if(options.cast !== true){ - throw new CsvError('CSV_INVALID_OPTION_CAST', [ - 'Invalid option cast:', 'cast must be true or a function,', - `got ${JSON.stringify(options.cast)}` - ], options); - } - // Normalize option `cast_date` - if(options.cast_date === undefined || options.cast_date === null || options.cast_date === false || options.cast_date === ''){ - options.cast_date = false; - }else if(options.cast_date === true){ - options.cast_date = function(value){ - const date = Date.parse(value); - return !isNaN(date) ? new Date(date) : value; - }; - }else { - throw new CsvError('CSV_INVALID_OPTION_CAST_DATE', [ - 'Invalid option cast_date:', 'cast_date must be true or a function,', - `got ${JSON.stringify(options.cast_date)}` - ], options); - } - // Normalize option `columns` - let fnFirstLineToHeaders = null; - if(options.columns === true){ - // Fields in the first line are converted as-is to columns - fnFirstLineToHeaders = undefined; - }else if(typeof options.columns === 'function'){ - fnFirstLineToHeaders = options.columns; - options.columns = true; - }else if(Array.isArray(options.columns)){ - options.columns = normalizeColumnsArray(options.columns); - }else if(options.columns === undefined || options.columns === null || options.columns === false){ - options.columns = false; - }else { - throw new CsvError('CSV_INVALID_OPTION_COLUMNS', [ - 'Invalid option columns:', - 'expect an array, a function or true,', - `got ${JSON.stringify(options.columns)}` +const normalize_options = function(opts){ + const options = {}; + // Merge with user options + for(const opt in opts){ + options[underscore(opt)] = opts[opt]; + } + // Normalize option `encoding` + // Note: defined first because other options depends on it + // to convert chars/strings into buffers. + if(options.encoding === undefined || options.encoding === true){ + options.encoding = 'utf8'; + }else if(options.encoding === null || options.encoding === false){ + options.encoding = null; + }else if(typeof options.encoding !== 'string' && options.encoding !== null){ + throw new CsvError('CSV_INVALID_OPTION_ENCODING', [ + 'Invalid option encoding:', + 'encoding must be a string or null to return a buffer,', + `got ${JSON.stringify(options.encoding)}` + ], options); + } + // Normalize option `bom` + if(options.bom === undefined || options.bom === null || options.bom === false){ + options.bom = false; + }else if(options.bom !== true){ + throw new CsvError('CSV_INVALID_OPTION_BOM', [ + 'Invalid option bom:', 'bom must be true,', + `got ${JSON.stringify(options.bom)}` + ], options); + } + // Normalize option `cast` + options.cast_function = null; + if(options.cast === undefined || options.cast === null || options.cast === false || options.cast === ''){ + options.cast = undefined; + }else if(typeof options.cast === 'function'){ + options.cast_function = options.cast; + options.cast = true; + }else if(options.cast !== true){ + throw new CsvError('CSV_INVALID_OPTION_CAST', [ + 'Invalid option cast:', 'cast must be true or a function,', + `got ${JSON.stringify(options.cast)}` + ], options); + } + // Normalize option `cast_date` + if(options.cast_date === undefined || options.cast_date === null || options.cast_date === false || options.cast_date === ''){ + options.cast_date = false; + }else if(options.cast_date === true){ + options.cast_date = function(value){ + const date = Date.parse(value); + return !isNaN(date) ? new Date(date) : value; + }; + }else { + throw new CsvError('CSV_INVALID_OPTION_CAST_DATE', [ + 'Invalid option cast_date:', 'cast_date must be true or a function,', + `got ${JSON.stringify(options.cast_date)}` + ], options); + } + // Normalize option `columns` + options.cast_first_line_to_header = null; + if(options.columns === true){ + // Fields in the first line are converted as-is to columns + options.cast_first_line_to_header = undefined; + }else if(typeof options.columns === 'function'){ + options.cast_first_line_to_header = options.columns; + options.columns = true; + }else if(Array.isArray(options.columns)){ + options.columns = normalize_columns_array(options.columns); + }else if(options.columns === undefined || options.columns === null || options.columns === false){ + options.columns = false; + }else { + throw new CsvError('CSV_INVALID_OPTION_COLUMNS', [ + 'Invalid option columns:', + 'expect an array, a function or true,', + `got ${JSON.stringify(options.columns)}` + ], options); + } + // Normalize option `group_columns_by_name` + if(options.group_columns_by_name === undefined || options.group_columns_by_name === null || options.group_columns_by_name === false){ + options.group_columns_by_name = false; + }else if(options.group_columns_by_name !== true){ + throw new CsvError('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ + 'Invalid option group_columns_by_name:', + 'expect an boolean,', + `got ${JSON.stringify(options.group_columns_by_name)}` + ], options); + }else if(options.columns === false){ + throw new CsvError('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ + 'Invalid option group_columns_by_name:', + 'the `columns` mode must be activated.' + ], options); + } + // Normalize option `comment` + if(options.comment === undefined || options.comment === null || options.comment === false || options.comment === ''){ + options.comment = null; + }else { + if(typeof options.comment === 'string'){ + options.comment = Buffer.from(options.comment, options.encoding); + } + if(!isBuffer(options.comment)){ + throw new CsvError('CSV_INVALID_OPTION_COMMENT', [ + 'Invalid option comment:', + 'comment must be a buffer or a string,', + `got ${JSON.stringify(options.comment)}` ], options); } - // Normalize option `group_columns_by_name` - if(options.group_columns_by_name === undefined || options.group_columns_by_name === null || options.group_columns_by_name === false){ - options.group_columns_by_name = false; - }else if(options.group_columns_by_name !== true){ - throw new CsvError('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ - 'Invalid option group_columns_by_name:', - 'expect an boolean,', - `got ${JSON.stringify(options.group_columns_by_name)}` - ], options); - }else if(options.columns === false){ - throw new CsvError('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ - 'Invalid option group_columns_by_name:', - 'the `columns` mode must be activated.' - ], options); + } + // Normalize option `delimiter` + const delimiter_json = JSON.stringify(options.delimiter); + if(!Array.isArray(options.delimiter)) options.delimiter = [options.delimiter]; + if(options.delimiter.length === 0){ + throw new CsvError('CSV_INVALID_OPTION_DELIMITER', [ + 'Invalid option delimiter:', + 'delimiter must be a non empty string or buffer or array of string|buffer,', + `got ${delimiter_json}` + ], options); + } + options.delimiter = options.delimiter.map(function(delimiter){ + if(delimiter === undefined || delimiter === null || delimiter === false){ + return Buffer.from(',', options.encoding); } - // Normalize option `comment` - if(options.comment === undefined || options.comment === null || options.comment === false || options.comment === ''){ - options.comment = null; - }else { - if(typeof options.comment === 'string'){ - options.comment = Buffer.from(options.comment, options.encoding); - } - if(!isBuffer(options.comment)){ - throw new CsvError('CSV_INVALID_OPTION_COMMENT', [ - 'Invalid option comment:', - 'comment must be a buffer or a string,', - `got ${JSON.stringify(options.comment)}` - ], options); - } + if(typeof delimiter === 'string'){ + delimiter = Buffer.from(delimiter, options.encoding); } - // Normalize option `delimiter` - const delimiter_json = JSON.stringify(options.delimiter); - if(!Array.isArray(options.delimiter)) options.delimiter = [options.delimiter]; - if(options.delimiter.length === 0){ + if(!isBuffer(delimiter) || delimiter.length === 0){ throw new CsvError('CSV_INVALID_OPTION_DELIMITER', [ 'Invalid option delimiter:', 'delimiter must be a non empty string or buffer or array of string|buffer,', `got ${delimiter_json}` ], options); } - options.delimiter = options.delimiter.map(function(delimiter){ - if(delimiter === undefined || delimiter === null || delimiter === false){ - return Buffer.from(',', options.encoding); - } - if(typeof delimiter === 'string'){ - delimiter = Buffer.from(delimiter, options.encoding); - } - if(!isBuffer(delimiter) || delimiter.length === 0){ - throw new CsvError('CSV_INVALID_OPTION_DELIMITER', [ - 'Invalid option delimiter:', - 'delimiter must be a non empty string or buffer or array of string|buffer,', - `got ${delimiter_json}` - ], options); - } - return delimiter; - }); - // Normalize option `escape` - if(options.escape === undefined || options.escape === true){ - options.escape = Buffer.from('"', options.encoding); - }else if(typeof options.escape === 'string'){ - options.escape = Buffer.from(options.escape, options.encoding); - }else if (options.escape === null || options.escape === false){ - options.escape = null; - } - if(options.escape !== null){ - if(!isBuffer(options.escape)){ - throw new Error(`Invalid Option: escape must be a buffer, a string or a boolean, got ${JSON.stringify(options.escape)}`); - } - } - // Normalize option `from` - if(options.from === undefined || options.from === null){ - options.from = 1; - }else { - if(typeof options.from === 'string' && /\d+/.test(options.from)){ - options.from = parseInt(options.from); - } - if(Number.isInteger(options.from)){ - if(options.from < 0){ - throw new Error(`Invalid Option: from must be a positive integer, got ${JSON.stringify(opts.from)}`); - } - }else { - throw new Error(`Invalid Option: from must be an integer, got ${JSON.stringify(options.from)}`); - } - } - // Normalize option `from_line` - if(options.from_line === undefined || options.from_line === null){ - options.from_line = 1; - }else { - if(typeof options.from_line === 'string' && /\d+/.test(options.from_line)){ - options.from_line = parseInt(options.from_line); - } - if(Number.isInteger(options.from_line)){ - if(options.from_line <= 0){ - throw new Error(`Invalid Option: from_line must be a positive integer greater than 0, got ${JSON.stringify(opts.from_line)}`); - } - }else { - throw new Error(`Invalid Option: from_line must be an integer, got ${JSON.stringify(opts.from_line)}`); - } - } - // Normalize options `ignore_last_delimiters` - if(options.ignore_last_delimiters === undefined || options.ignore_last_delimiters === null){ - options.ignore_last_delimiters = false; - }else if(typeof options.ignore_last_delimiters === 'number'){ - options.ignore_last_delimiters = Math.floor(options.ignore_last_delimiters); - if(options.ignore_last_delimiters === 0){ - options.ignore_last_delimiters = false; - } - }else if(typeof options.ignore_last_delimiters !== 'boolean'){ - throw new CsvError('CSV_INVALID_OPTION_IGNORE_LAST_DELIMITERS', [ - 'Invalid option `ignore_last_delimiters`:', - 'the value must be a boolean value or an integer,', - `got ${JSON.stringify(options.ignore_last_delimiters)}` - ], options); - } - if(options.ignore_last_delimiters === true && options.columns === false){ - throw new CsvError('CSV_IGNORE_LAST_DELIMITERS_REQUIRES_COLUMNS', [ - 'The option `ignore_last_delimiters`', - 'requires the activation of the `columns` option' - ], options); - } - // Normalize option `info` - if(options.info === undefined || options.info === null || options.info === false){ - options.info = false; - }else if(options.info !== true){ - throw new Error(`Invalid Option: info must be true, got ${JSON.stringify(options.info)}`); - } - // Normalize option `max_record_size` - if(options.max_record_size === undefined || options.max_record_size === null || options.max_record_size === false){ - options.max_record_size = 0; - }else if(Number.isInteger(options.max_record_size) && options.max_record_size >= 0);else if(typeof options.max_record_size === 'string' && /\d+/.test(options.max_record_size)){ - options.max_record_size = parseInt(options.max_record_size); - }else { - throw new Error(`Invalid Option: max_record_size must be a positive integer, got ${JSON.stringify(options.max_record_size)}`); - } - // Normalize option `objname` - if(options.objname === undefined || options.objname === null || options.objname === false){ - options.objname = undefined; - }else if(isBuffer(options.objname)){ - if(options.objname.length === 0){ - throw new Error(`Invalid Option: objname must be a non empty buffer`); - } - if(options.encoding === null);else { - options.objname = options.objname.toString(options.encoding); - } - }else if(typeof options.objname === 'string'){ - if(options.objname.length === 0){ - throw new Error(`Invalid Option: objname must be a non empty string`); - } - // Great, nothing to do - }else if(typeof options.objname === 'number');else { - throw new Error(`Invalid Option: objname must be a string or a buffer, got ${options.objname}`); - } - if(options.objname !== undefined){ - if(typeof options.objname === 'number'){ - if(options.columns !== false){ - throw Error('Invalid Option: objname index cannot be combined with columns or be defined as a field'); - } - }else { // A string or a buffer - if(options.columns === false){ - throw Error('Invalid Option: objname field must be combined with columns or be defined as an index'); - } - } - } - // Normalize option `on_record` - if(options.on_record === undefined || options.on_record === null){ - options.on_record = undefined; - }else if(typeof options.on_record !== 'function'){ - throw new CsvError('CSV_INVALID_OPTION_ON_RECORD', [ - 'Invalid option `on_record`:', - 'expect a function,', - `got ${JSON.stringify(options.on_record)}` - ], options); - } - // Normalize option `quote` - if(options.quote === null || options.quote === false || options.quote === ''){ - options.quote = null; - }else { - if(options.quote === undefined || options.quote === true){ - options.quote = Buffer.from('"', options.encoding); - }else if(typeof options.quote === 'string'){ - options.quote = Buffer.from(options.quote, options.encoding); - } - if(!isBuffer(options.quote)){ - throw new Error(`Invalid Option: quote must be a buffer or a string, got ${JSON.stringify(options.quote)}`); - } - } - // Normalize option `raw` - if(options.raw === undefined || options.raw === null || options.raw === false){ - options.raw = false; - }else if(options.raw !== true){ - throw new Error(`Invalid Option: raw must be true, got ${JSON.stringify(options.raw)}`); - } - // Normalize option `record_delimiter` - if(options.record_delimiter === undefined){ - options.record_delimiter = []; - }else if(typeof options.record_delimiter === 'string' || isBuffer(options.record_delimiter)){ - if(options.record_delimiter.length === 0){ - throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ - 'Invalid option `record_delimiter`:', - 'value must be a non empty string or buffer,', - `got ${JSON.stringify(options.record_delimiter)}` - ], options); - } - options.record_delimiter = [options.record_delimiter]; - }else if(!Array.isArray(options.record_delimiter)){ - throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ - 'Invalid option `record_delimiter`:', - 'value must be a string, a buffer or array of string|buffer,', - `got ${JSON.stringify(options.record_delimiter)}` - ], options); - } - options.record_delimiter = options.record_delimiter.map(function(rd, i){ - if(typeof rd !== 'string' && ! isBuffer(rd)){ - throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ - 'Invalid option `record_delimiter`:', - 'value must be a string, a buffer or array of string|buffer', - `at index ${i},`, - `got ${JSON.stringify(rd)}` - ], options); - }else if(rd.length === 0){ - throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ - 'Invalid option `record_delimiter`:', - 'value must be a non empty string or buffer', - `at index ${i},`, - `got ${JSON.stringify(rd)}` - ], options); - } - if(typeof rd === 'string'){ - rd = Buffer.from(rd, options.encoding); + return delimiter; + }); + // Normalize option `escape` + if(options.escape === undefined || options.escape === true){ + options.escape = Buffer.from('"', options.encoding); + }else if(typeof options.escape === 'string'){ + options.escape = Buffer.from(options.escape, options.encoding); + }else if (options.escape === null || options.escape === false){ + options.escape = null; + } + if(options.escape !== null){ + if(!isBuffer(options.escape)){ + throw new Error(`Invalid Option: escape must be a buffer, a string or a boolean, got ${JSON.stringify(options.escape)}`); + } + } + // Normalize option `from` + if(options.from === undefined || options.from === null){ + options.from = 1; + }else { + if(typeof options.from === 'string' && /\d+/.test(options.from)){ + options.from = parseInt(options.from); + } + if(Number.isInteger(options.from)){ + if(options.from < 0){ + throw new Error(`Invalid Option: from must be a positive integer, got ${JSON.stringify(opts.from)}`); } - return rd; - }); - // Normalize option `relax_column_count` - if(typeof options.relax_column_count === 'boolean');else if(options.relax_column_count === undefined || options.relax_column_count === null){ - options.relax_column_count = false; - }else { - throw new Error(`Invalid Option: relax_column_count must be a boolean, got ${JSON.stringify(options.relax_column_count)}`); - } - if(typeof options.relax_column_count_less === 'boolean');else if(options.relax_column_count_less === undefined || options.relax_column_count_less === null){ - options.relax_column_count_less = false; - }else { - throw new Error(`Invalid Option: relax_column_count_less must be a boolean, got ${JSON.stringify(options.relax_column_count_less)}`); - } - if(typeof options.relax_column_count_more === 'boolean');else if(options.relax_column_count_more === undefined || options.relax_column_count_more === null){ - options.relax_column_count_more = false; - }else { - throw new Error(`Invalid Option: relax_column_count_more must be a boolean, got ${JSON.stringify(options.relax_column_count_more)}`); - } - // Normalize option `relax_quotes` - if(typeof options.relax_quotes === 'boolean');else if(options.relax_quotes === undefined || options.relax_quotes === null){ - options.relax_quotes = false; - }else { - throw new Error(`Invalid Option: relax_quotes must be a boolean, got ${JSON.stringify(options.relax_quotes)}`); - } - // Normalize option `skip_empty_lines` - if(typeof options.skip_empty_lines === 'boolean');else if(options.skip_empty_lines === undefined || options.skip_empty_lines === null){ - options.skip_empty_lines = false; }else { - throw new Error(`Invalid Option: skip_empty_lines must be a boolean, got ${JSON.stringify(options.skip_empty_lines)}`); + throw new Error(`Invalid Option: from must be an integer, got ${JSON.stringify(options.from)}`); } - // Normalize option `skip_records_with_empty_values` - if(typeof options.skip_records_with_empty_values === 'boolean');else if(options.skip_records_with_empty_values === undefined || options.skip_records_with_empty_values === null){ - options.skip_records_with_empty_values = false; - }else { - throw new Error(`Invalid Option: skip_records_with_empty_values must be a boolean, got ${JSON.stringify(options.skip_records_with_empty_values)}`); + } + // Normalize option `from_line` + if(options.from_line === undefined || options.from_line === null){ + options.from_line = 1; + }else { + if(typeof options.from_line === 'string' && /\d+/.test(options.from_line)){ + options.from_line = parseInt(options.from_line); } - // Normalize option `skip_records_with_error` - if(typeof options.skip_records_with_error === 'boolean');else if(options.skip_records_with_error === undefined || options.skip_records_with_error === null){ - options.skip_records_with_error = false; + if(Number.isInteger(options.from_line)){ + if(options.from_line <= 0){ + throw new Error(`Invalid Option: from_line must be a positive integer greater than 0, got ${JSON.stringify(opts.from_line)}`); + } }else { - throw new Error(`Invalid Option: skip_records_with_error must be a boolean, got ${JSON.stringify(options.skip_records_with_error)}`); - } - // Normalize option `rtrim` - if(options.rtrim === undefined || options.rtrim === null || options.rtrim === false){ - options.rtrim = false; - }else if(options.rtrim !== true){ - throw new Error(`Invalid Option: rtrim must be a boolean, got ${JSON.stringify(options.rtrim)}`); + throw new Error(`Invalid Option: from_line must be an integer, got ${JSON.stringify(opts.from_line)}`); } - // Normalize option `ltrim` - if(options.ltrim === undefined || options.ltrim === null || options.ltrim === false){ - options.ltrim = false; - }else if(options.ltrim !== true){ - throw new Error(`Invalid Option: ltrim must be a boolean, got ${JSON.stringify(options.ltrim)}`); + } + // Normalize options `ignore_last_delimiters` + if(options.ignore_last_delimiters === undefined || options.ignore_last_delimiters === null){ + options.ignore_last_delimiters = false; + }else if(typeof options.ignore_last_delimiters === 'number'){ + options.ignore_last_delimiters = Math.floor(options.ignore_last_delimiters); + if(options.ignore_last_delimiters === 0){ + options.ignore_last_delimiters = false; } - // Normalize option `trim` - if(options.trim === undefined || options.trim === null || options.trim === false){ - options.trim = false; - }else if(options.trim !== true){ - throw new Error(`Invalid Option: trim must be a boolean, got ${JSON.stringify(options.trim)}`); + }else if(typeof options.ignore_last_delimiters !== 'boolean'){ + throw new CsvError('CSV_INVALID_OPTION_IGNORE_LAST_DELIMITERS', [ + 'Invalid option `ignore_last_delimiters`:', + 'the value must be a boolean value or an integer,', + `got ${JSON.stringify(options.ignore_last_delimiters)}` + ], options); + } + if(options.ignore_last_delimiters === true && options.columns === false){ + throw new CsvError('CSV_IGNORE_LAST_DELIMITERS_REQUIRES_COLUMNS', [ + 'The option `ignore_last_delimiters`', + 'requires the activation of the `columns` option' + ], options); + } + // Normalize option `info` + if(options.info === undefined || options.info === null || options.info === false){ + options.info = false; + }else if(options.info !== true){ + throw new Error(`Invalid Option: info must be true, got ${JSON.stringify(options.info)}`); + } + // Normalize option `max_record_size` + if(options.max_record_size === undefined || options.max_record_size === null || options.max_record_size === false){ + options.max_record_size = 0; + }else if(Number.isInteger(options.max_record_size) && options.max_record_size >= 0);else if(typeof options.max_record_size === 'string' && /\d+/.test(options.max_record_size)){ + options.max_record_size = parseInt(options.max_record_size); + }else { + throw new Error(`Invalid Option: max_record_size must be a positive integer, got ${JSON.stringify(options.max_record_size)}`); + } + // Normalize option `objname` + if(options.objname === undefined || options.objname === null || options.objname === false){ + options.objname = undefined; + }else if(isBuffer(options.objname)){ + if(options.objname.length === 0){ + throw new Error(`Invalid Option: objname must be a non empty buffer`); + } + if(options.encoding === null);else { + options.objname = options.objname.toString(options.encoding); + } + }else if(typeof options.objname === 'string'){ + if(options.objname.length === 0){ + throw new Error(`Invalid Option: objname must be a non empty string`); + } + // Great, nothing to do + }else if(typeof options.objname === 'number');else { + throw new Error(`Invalid Option: objname must be a string or a buffer, got ${options.objname}`); + } + if(options.objname !== undefined){ + if(typeof options.objname === 'number'){ + if(options.columns !== false){ + throw Error('Invalid Option: objname index cannot be combined with columns or be defined as a field'); + } + }else { // A string or a buffer + if(options.columns === false){ + throw Error('Invalid Option: objname field must be combined with columns or be defined as an index'); + } + } + } + // Normalize option `on_record` + if(options.on_record === undefined || options.on_record === null){ + options.on_record = undefined; + }else if(typeof options.on_record !== 'function'){ + throw new CsvError('CSV_INVALID_OPTION_ON_RECORD', [ + 'Invalid option `on_record`:', + 'expect a function,', + `got ${JSON.stringify(options.on_record)}` + ], options); + } + // Normalize option `quote` + if(options.quote === null || options.quote === false || options.quote === ''){ + options.quote = null; + }else { + if(options.quote === undefined || options.quote === true){ + options.quote = Buffer.from('"', options.encoding); + }else if(typeof options.quote === 'string'){ + options.quote = Buffer.from(options.quote, options.encoding); + } + if(!isBuffer(options.quote)){ + throw new Error(`Invalid Option: quote must be a buffer or a string, got ${JSON.stringify(options.quote)}`); + } + } + // Normalize option `raw` + if(options.raw === undefined || options.raw === null || options.raw === false){ + options.raw = false; + }else if(options.raw !== true){ + throw new Error(`Invalid Option: raw must be true, got ${JSON.stringify(options.raw)}`); + } + // Normalize option `record_delimiter` + if(options.record_delimiter === undefined){ + options.record_delimiter = []; + }else if(typeof options.record_delimiter === 'string' || isBuffer(options.record_delimiter)){ + if(options.record_delimiter.length === 0){ + throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ + 'Invalid option `record_delimiter`:', + 'value must be a non empty string or buffer,', + `got ${JSON.stringify(options.record_delimiter)}` + ], options); } - // Normalize options `trim`, `ltrim` and `rtrim` - if(options.trim === true && opts.ltrim !== false){ - options.ltrim = true; - }else if(options.ltrim !== true){ - options.ltrim = false; + options.record_delimiter = [options.record_delimiter]; + }else if(!Array.isArray(options.record_delimiter)){ + throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ + 'Invalid option `record_delimiter`:', + 'value must be a string, a buffer or array of string|buffer,', + `got ${JSON.stringify(options.record_delimiter)}` + ], options); + } + options.record_delimiter = options.record_delimiter.map(function(rd, i){ + if(typeof rd !== 'string' && ! isBuffer(rd)){ + throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ + 'Invalid option `record_delimiter`:', + 'value must be a string, a buffer or array of string|buffer', + `at index ${i},`, + `got ${JSON.stringify(rd)}` + ], options); + }else if(rd.length === 0){ + throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ + 'Invalid option `record_delimiter`:', + 'value must be a non empty string or buffer', + `at index ${i},`, + `got ${JSON.stringify(rd)}` + ], options); } - if(options.trim === true && opts.rtrim !== false){ - options.rtrim = true; - }else if(options.rtrim !== true){ - options.rtrim = false; + if(typeof rd === 'string'){ + rd = Buffer.from(rd, options.encoding); } - // Normalize option `to` - if(options.to === undefined || options.to === null){ - options.to = -1; - }else { - if(typeof options.to === 'string' && /\d+/.test(options.to)){ - options.to = parseInt(options.to); - } - if(Number.isInteger(options.to)){ - if(options.to <= 0){ - throw new Error(`Invalid Option: to must be a positive integer greater than 0, got ${JSON.stringify(opts.to)}`); - } - }else { - throw new Error(`Invalid Option: to must be an integer, got ${JSON.stringify(opts.to)}`); + return rd; + }); + // Normalize option `relax_column_count` + if(typeof options.relax_column_count === 'boolean');else if(options.relax_column_count === undefined || options.relax_column_count === null){ + options.relax_column_count = false; + }else { + throw new Error(`Invalid Option: relax_column_count must be a boolean, got ${JSON.stringify(options.relax_column_count)}`); + } + if(typeof options.relax_column_count_less === 'boolean');else if(options.relax_column_count_less === undefined || options.relax_column_count_less === null){ + options.relax_column_count_less = false; + }else { + throw new Error(`Invalid Option: relax_column_count_less must be a boolean, got ${JSON.stringify(options.relax_column_count_less)}`); + } + if(typeof options.relax_column_count_more === 'boolean');else if(options.relax_column_count_more === undefined || options.relax_column_count_more === null){ + options.relax_column_count_more = false; + }else { + throw new Error(`Invalid Option: relax_column_count_more must be a boolean, got ${JSON.stringify(options.relax_column_count_more)}`); + } + // Normalize option `relax_quotes` + if(typeof options.relax_quotes === 'boolean');else if(options.relax_quotes === undefined || options.relax_quotes === null){ + options.relax_quotes = false; + }else { + throw new Error(`Invalid Option: relax_quotes must be a boolean, got ${JSON.stringify(options.relax_quotes)}`); + } + // Normalize option `skip_empty_lines` + if(typeof options.skip_empty_lines === 'boolean');else if(options.skip_empty_lines === undefined || options.skip_empty_lines === null){ + options.skip_empty_lines = false; + }else { + throw new Error(`Invalid Option: skip_empty_lines must be a boolean, got ${JSON.stringify(options.skip_empty_lines)}`); + } + // Normalize option `skip_records_with_empty_values` + if(typeof options.skip_records_with_empty_values === 'boolean');else if(options.skip_records_with_empty_values === undefined || options.skip_records_with_empty_values === null){ + options.skip_records_with_empty_values = false; + }else { + throw new Error(`Invalid Option: skip_records_with_empty_values must be a boolean, got ${JSON.stringify(options.skip_records_with_empty_values)}`); + } + // Normalize option `skip_records_with_error` + if(typeof options.skip_records_with_error === 'boolean');else if(options.skip_records_with_error === undefined || options.skip_records_with_error === null){ + options.skip_records_with_error = false; + }else { + throw new Error(`Invalid Option: skip_records_with_error must be a boolean, got ${JSON.stringify(options.skip_records_with_error)}`); + } + // Normalize option `rtrim` + if(options.rtrim === undefined || options.rtrim === null || options.rtrim === false){ + options.rtrim = false; + }else if(options.rtrim !== true){ + throw new Error(`Invalid Option: rtrim must be a boolean, got ${JSON.stringify(options.rtrim)}`); + } + // Normalize option `ltrim` + if(options.ltrim === undefined || options.ltrim === null || options.ltrim === false){ + options.ltrim = false; + }else if(options.ltrim !== true){ + throw new Error(`Invalid Option: ltrim must be a boolean, got ${JSON.stringify(options.ltrim)}`); + } + // Normalize option `trim` + if(options.trim === undefined || options.trim === null || options.trim === false){ + options.trim = false; + }else if(options.trim !== true){ + throw new Error(`Invalid Option: trim must be a boolean, got ${JSON.stringify(options.trim)}`); + } + // Normalize options `trim`, `ltrim` and `rtrim` + if(options.trim === true && opts.ltrim !== false){ + options.ltrim = true; + }else if(options.ltrim !== true){ + options.ltrim = false; + } + if(options.trim === true && opts.rtrim !== false){ + options.rtrim = true; + }else if(options.rtrim !== true){ + options.rtrim = false; + } + // Normalize option `to` + if(options.to === undefined || options.to === null){ + options.to = -1; + }else { + if(typeof options.to === 'string' && /\d+/.test(options.to)){ + options.to = parseInt(options.to); + } + if(Number.isInteger(options.to)){ + if(options.to <= 0){ + throw new Error(`Invalid Option: to must be a positive integer greater than 0, got ${JSON.stringify(opts.to)}`); } - } - // Normalize option `to_line` - if(options.to_line === undefined || options.to_line === null){ - options.to_line = -1; }else { - if(typeof options.to_line === 'string' && /\d+/.test(options.to_line)){ - options.to_line = parseInt(options.to_line); - } - if(Number.isInteger(options.to_line)){ - if(options.to_line <= 0){ - throw new Error(`Invalid Option: to_line must be a positive integer greater than 0, got ${JSON.stringify(opts.to_line)}`); - } - }else { - throw new Error(`Invalid Option: to_line must be an integer, got ${JSON.stringify(opts.to_line)}`); - } - } - this.info = { - bytes: 0, - comment_lines: 0, - empty_lines: 0, - invalid_field_length: 0, - lines: 1, - records: 0 - }; - this.options = options; - this.state = { - bomSkipped: false, - bufBytesStart: 0, - castField: fnCastField, - commenting: false, - // Current error encountered by a record - error: undefined, - enabled: options.from_line === 1, - escaping: false, - escapeIsQuote: isBuffer(options.escape) && isBuffer(options.quote) && Buffer.compare(options.escape, options.quote) === 0, - // columns can be `false`, `true`, `Array` - expectedRecordLength: Array.isArray(options.columns) ? options.columns.length : undefined, - field: new ResizeableBuffer(20), - firstLineToHeaders: fnFirstLineToHeaders, - needMoreDataSize: Math.max( - // Skip if the remaining buffer smaller than comment - options.comment !== null ? options.comment.length : 0, - // Skip if the remaining buffer can be delimiter - ...options.delimiter.map((delimiter) => delimiter.length), - // Skip if the remaining buffer can be escape sequence - options.quote !== null ? options.quote.length : 0, - ), - previousBuf: undefined, - quoting: false, - stop: false, - rawBuffer: new ResizeableBuffer(100), - record: [], - recordHasError: false, - record_length: 0, - recordDelimiterMaxLength: options.record_delimiter.length === 0 ? 2 : Math.max(...options.record_delimiter.map((v) => v.length)), - trimChars: [Buffer.from(' ', options.encoding)[0], Buffer.from('\t', options.encoding)[0]], - wasQuoting: false, - wasRowDelimiter: false - }; - } - // Implementation of `Transform._transform` - _transform(buf, encoding, callback){ - if(this.state.stop === true){ - return; - } - const err = this.__parse(buf, false); - if(err !== undefined){ - this.state.stop = true; + throw new Error(`Invalid Option: to must be an integer, got ${JSON.stringify(opts.to)}`); } - callback(err); } - // Implementation of `Transform._flush` - _flush(callback){ - if(this.state.stop === true){ - return; + // Normalize option `to_line` + if(options.to_line === undefined || options.to_line === null){ + options.to_line = -1; + }else { + if(typeof options.to_line === 'string' && /\d+/.test(options.to_line)){ + options.to_line = parseInt(options.to_line); } - const err = this.__parse(undefined, true); - callback(err); - } - // Central parser implementation - __parse(nextBuf, end){ - const {bom, comment, escape, from_line, ltrim, max_record_size, quote, raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options; - let {record_delimiter} = this.options; - const {bomSkipped, previousBuf, rawBuffer, escapeIsQuote} = this.state; - let buf; - if(previousBuf === undefined){ - if(nextBuf === undefined){ - // Handle empty string - this.push(null); - return; - }else { - buf = nextBuf; + if(Number.isInteger(options.to_line)){ + if(options.to_line <= 0){ + throw new Error(`Invalid Option: to_line must be a positive integer greater than 0, got ${JSON.stringify(opts.to_line)}`); } - }else if(previousBuf !== undefined && nextBuf === undefined){ - buf = previousBuf; }else { - buf = Buffer.concat([previousBuf, nextBuf]); + throw new Error(`Invalid Option: to_line must be an integer, got ${JSON.stringify(opts.to_line)}`); } - // Handle UTF BOM - if(bomSkipped === false){ - if(bom === false){ - this.state.bomSkipped = true; - }else if(buf.length < 3){ - // No enough data - if(end === false){ - // Wait for more data - this.state.previousBuf = buf; + } + return options; +}; + +const isRecordEmpty = function(record){ + return record.every((field) => field == null || field.toString && field.toString().trim() === ''); +}; + +// white space characters +// https://en.wikipedia.org/wiki/Whitespace_character +// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions/Character_Classes#Types +// \f\n\r\t\v\u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff +const tab = 9; +const nl = 10; // \n, 0x0A in hexadecimal, 10 in decimal +const np = 12; +const cr = 13; // \r, 0x0D in hexadécimal, 13 in decimal +const space = 32; +const boms = { + // Note, the following are equals: + // Buffer.from("\ufeff") + // Buffer.from([239, 187, 191]) + // Buffer.from('EFBBBF', 'hex') + 'utf8': Buffer.from([239, 187, 191]), + // Note, the following are equals: + // Buffer.from "\ufeff", 'utf16le + // Buffer.from([255, 254]) + 'utf16le': Buffer.from([255, 254]) +}; + +const transform = function(original_options = {}) { + const info = { + bytes: 0, + comment_lines: 0, + empty_lines: 0, + invalid_field_length: 0, + lines: 1, + records: 0 + }; + const options = normalize_options(original_options); + return { + info: info, + original_options: original_options, + options: options, + state: init_state(options), + __needMoreData: function(i, bufLen, end){ + if(end) return false; + const {quote} = this.options; + const {quoting, needMoreDataSize, recordDelimiterMaxLength} = this.state; + const numOfCharLeft = bufLen - i - 1; + const requiredLength = Math.max( + needMoreDataSize, + // Skip if the remaining buffer smaller than record delimiter + recordDelimiterMaxLength, + // Skip if the remaining buffer can be record delimiter following the closing quote + // 1 is for quote.length + quoting ? (quote.length + recordDelimiterMaxLength) : 0, + ); + return numOfCharLeft < requiredLength; + }, + // Central parser implementation + parse: function(nextBuf, end, push, close){ + const {bom, comment, escape, from_line, ltrim, max_record_size, quote, raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options; + let {record_delimiter} = this.options; + const {bomSkipped, previousBuf, rawBuffer, escapeIsQuote} = this.state; + let buf; + if(previousBuf === undefined){ + if(nextBuf === undefined){ + // Handle empty string + close(); return; + }else { + buf = nextBuf; } + }else if(previousBuf !== undefined && nextBuf === undefined){ + buf = previousBuf; }else { - for(const encoding in boms){ - if(boms[encoding].compare(buf, 0, boms[encoding].length) === 0){ - // Skip BOM - const bomLength = boms[encoding].length; - this.state.bufBytesStart += bomLength; - buf = buf.slice(bomLength); - // Renormalize original options with the new encoding - this.__normalizeOptions({...this.__originalOptions, encoding: encoding}); - break; + buf = Buffer.concat([previousBuf, nextBuf]); + } + // Handle UTF BOM + if(bomSkipped === false){ + if(bom === false){ + this.state.bomSkipped = true; + }else if(buf.length < 3){ + // No enough data + if(end === false){ + // Wait for more data + this.state.previousBuf = buf; + return; + } + }else { + for(const encoding in boms){ + if(boms[encoding].compare(buf, 0, boms[encoding].length) === 0){ + // Skip BOM + const bomLength = boms[encoding].length; + this.state.bufBytesStart += bomLength; + buf = buf.slice(bomLength); + // Renormalize original options with the new encoding + this.options = normalize_options({...this.original_options, encoding: encoding}); + break; + } } + this.state.bomSkipped = true; } - this.state.bomSkipped = true; - } - } - const bufLen = buf.length; - let pos; - for(pos = 0; pos < bufLen; pos++){ - // Ensure we get enough space to look ahead - // There should be a way to move this out of the loop - if(this.__needMoreData(pos, bufLen, end)){ - break; - } - if(this.state.wasRowDelimiter === true){ - this.info.lines++; - this.state.wasRowDelimiter = false; - } - if(to_line !== -1 && this.info.lines > to_line){ - this.state.stop = true; - this.push(null); - return; } - // Auto discovery of record_delimiter, unix, mac and windows supported - if(this.state.quoting === false && record_delimiter.length === 0){ - const record_delimiterCount = this.__autoDiscoverRecordDelimiter(buf, pos); - if(record_delimiterCount){ - record_delimiter = this.options.record_delimiter; + const bufLen = buf.length; + let pos; + for(pos = 0; pos < bufLen; pos++){ + // Ensure we get enough space to look ahead + // There should be a way to move this out of the loop + if(this.__needMoreData(pos, bufLen, end)){ + break; } - } - const chr = buf[pos]; - if(raw === true){ - rawBuffer.append(chr); - } - if((chr === cr || chr === nl) && this.state.wasRowDelimiter === false){ - this.state.wasRowDelimiter = true; - } - // Previous char was a valid escape char - // treat the current char as a regular char - if(this.state.escaping === true){ - this.state.escaping = false; - }else { - // Escape is only active inside quoted fields - // We are quoting, the char is an escape chr and there is a chr to escape - // if(escape !== null && this.state.quoting === true && chr === escape && pos + 1 < bufLen){ - if(escape !== null && this.state.quoting === true && this.__isEscape(buf, pos, chr) && pos + escape.length < bufLen){ - if(escapeIsQuote){ - if(this.__isQuote(buf, pos+escape.length)){ - this.state.escaping = true; - pos += escape.length - 1; - continue; - } - }else { - this.state.escaping = true; - pos += escape.length - 1; - continue; + if(this.state.wasRowDelimiter === true){ + this.info.lines++; + this.state.wasRowDelimiter = false; + } + if(to_line !== -1 && this.info.lines > to_line){ + this.state.stop = true; + close(); + return; + } + // Auto discovery of record_delimiter, unix, mac and windows supported + if(this.state.quoting === false && record_delimiter.length === 0){ + const record_delimiterCount = this.__autoDiscoverRecordDelimiter(buf, pos); + if(record_delimiterCount){ + record_delimiter = this.options.record_delimiter; } } - // Not currently escaping and chr is a quote - // TODO: need to compare bytes instead of single char - if(this.state.commenting === false && this.__isQuote(buf, pos)){ - if(this.state.quoting === true){ - const nextChr = buf[pos+quote.length]; - const isNextChrTrimable = rtrim && this.__isCharTrimable(nextChr); - const isNextChrComment = comment !== null && this.__compareBytes(comment, buf, pos+quote.length, nextChr); - const isNextChrDelimiter = this.__isDelimiter(buf, pos+quote.length, nextChr); - const isNextChrRecordDelimiter = record_delimiter.length === 0 ? this.__autoDiscoverRecordDelimiter(buf, pos+quote.length) : this.__isRecordDelimiter(nextChr, buf, pos+quote.length); - // Escape a quote - // Treat next char as a regular character - if(escape !== null && this.__isEscape(buf, pos, chr) && this.__isQuote(buf, pos + escape.length)){ + const chr = buf[pos]; + if(raw === true){ + rawBuffer.append(chr); + } + if((chr === cr || chr === nl) && this.state.wasRowDelimiter === false){ + this.state.wasRowDelimiter = true; + } + // Previous char was a valid escape char + // treat the current char as a regular char + if(this.state.escaping === true){ + this.state.escaping = false; + }else { + // Escape is only active inside quoted fields + // We are quoting, the char is an escape chr and there is a chr to escape + // if(escape !== null && this.state.quoting === true && chr === escape && pos + 1 < bufLen){ + if(escape !== null && this.state.quoting === true && this.__isEscape(buf, pos, chr) && pos + escape.length < bufLen){ + if(escapeIsQuote){ + if(this.__isQuote(buf, pos+escape.length)){ + this.state.escaping = true; + pos += escape.length - 1; + continue; + } + }else { + this.state.escaping = true; pos += escape.length - 1; - }else if(!nextChr || isNextChrDelimiter || isNextChrRecordDelimiter || isNextChrComment || isNextChrTrimable){ - this.state.quoting = false; - this.state.wasQuoting = true; - pos += quote.length - 1; continue; - }else if(relax_quotes === false){ - const err = this.__error( - new CsvError('CSV_INVALID_CLOSING_QUOTE', [ - 'Invalid Closing Quote:', - `got "${String.fromCharCode(nextChr)}"`, - `at line ${this.info.lines}`, - 'instead of delimiter, record delimiter, trimable character', - '(if activated) or comment', - ], this.options, this.__infoField()) - ); - if(err !== undefined) return err; - }else { - this.state.quoting = false; - this.state.wasQuoting = true; - this.state.field.prepend(quote); - pos += quote.length - 1; } - }else { - if(this.state.field.length !== 0){ - // In relax_quotes mode, treat opening quote preceded by chrs as regular - if(relax_quotes === false){ + } + // Not currently escaping and chr is a quote + // TODO: need to compare bytes instead of single char + if(this.state.commenting === false && this.__isQuote(buf, pos)){ + if(this.state.quoting === true){ + const nextChr = buf[pos+quote.length]; + const isNextChrTrimable = rtrim && this.__isCharTrimable(nextChr); + const isNextChrComment = comment !== null && this.__compareBytes(comment, buf, pos+quote.length, nextChr); + const isNextChrDelimiter = this.__isDelimiter(buf, pos+quote.length, nextChr); + const isNextChrRecordDelimiter = record_delimiter.length === 0 ? this.__autoDiscoverRecordDelimiter(buf, pos+quote.length) : this.__isRecordDelimiter(nextChr, buf, pos+quote.length); + // Escape a quote + // Treat next char as a regular character + if(escape !== null && this.__isEscape(buf, pos, chr) && this.__isQuote(buf, pos + escape.length)){ + pos += escape.length - 1; + }else if(!nextChr || isNextChrDelimiter || isNextChrRecordDelimiter || isNextChrComment || isNextChrTrimable){ + this.state.quoting = false; + this.state.wasQuoting = true; + pos += quote.length - 1; + continue; + }else if(relax_quotes === false){ const err = this.__error( - new CsvError('INVALID_OPENING_QUOTE', [ - 'Invalid Opening Quote:', - `a quote is found inside a field at line ${this.info.lines}`, - ], this.options, this.__infoField(), { - field: this.state.field, - }) + new CsvError('CSV_INVALID_CLOSING_QUOTE', [ + 'Invalid Closing Quote:', + `got "${String.fromCharCode(nextChr)}"`, + `at line ${this.info.lines}`, + 'instead of delimiter, record delimiter, trimable character', + '(if activated) or comment', + ], this.options, this.__infoField()) ); if(err !== undefined) return err; + }else { + this.state.quoting = false; + this.state.wasQuoting = true; + this.state.field.prepend(quote); + pos += quote.length - 1; } }else { - this.state.quoting = true; - pos += quote.length - 1; - continue; - } - } - } - if(this.state.quoting === false){ - const recordDelimiterLength = this.__isRecordDelimiter(chr, buf, pos); - if(recordDelimiterLength !== 0){ - // Do not emit comments which take a full line - const skipCommentLine = this.state.commenting && (this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0); - if(skipCommentLine){ - this.info.comment_lines++; - // Skip full comment line - }else { - // Activate records emition if above from_line - if(this.state.enabled === false && this.info.lines + (this.state.wasRowDelimiter === true ? 1: 0) >= from_line){ - this.state.enabled = true; - this.__resetField(); - this.__resetRecord(); - pos += recordDelimiterLength - 1; + if(this.state.field.length !== 0){ + // In relax_quotes mode, treat opening quote preceded by chrs as regular + if(relax_quotes === false){ + const err = this.__error( + new CsvError('INVALID_OPENING_QUOTE', [ + 'Invalid Opening Quote:', + `a quote is found inside a field at line ${this.info.lines}`, + ], this.options, this.__infoField(), { + field: this.state.field, + }) + ); + if(err !== undefined) return err; + } + }else { + this.state.quoting = true; + pos += quote.length - 1; continue; } - // Skip if line is empty and skip_empty_lines activated - if(skip_empty_lines === true && this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0){ - this.info.empty_lines++; - pos += recordDelimiterLength - 1; - continue; + } + } + if(this.state.quoting === false){ + const recordDelimiterLength = this.__isRecordDelimiter(chr, buf, pos); + if(recordDelimiterLength !== 0){ + // Do not emit comments which take a full line + const skipCommentLine = this.state.commenting && (this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0); + if(skipCommentLine){ + this.info.comment_lines++; + // Skip full comment line + }else { + // Activate records emition if above from_line + if(this.state.enabled === false && this.info.lines + (this.state.wasRowDelimiter === true ? 1: 0) >= from_line){ + this.state.enabled = true; + this.__resetField(); + this.__resetRecord(); + pos += recordDelimiterLength - 1; + continue; + } + // Skip if line is empty and skip_empty_lines activated + if(skip_empty_lines === true && this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0){ + this.info.empty_lines++; + pos += recordDelimiterLength - 1; + continue; + } + this.info.bytes = this.state.bufBytesStart + pos; + const errField = this.__onField(); + if(errField !== undefined) return errField; + this.info.bytes = this.state.bufBytesStart + pos + recordDelimiterLength; + const errRecord = this.__onRecord(push); + if(errRecord !== undefined) return errRecord; + if(to !== -1 && this.info.records >= to){ + this.state.stop = true; + close(); + return; + } } + this.state.commenting = false; + pos += recordDelimiterLength - 1; + continue; + } + if(this.state.commenting){ + continue; + } + const commentCount = comment === null ? 0 : this.__compareBytes(comment, buf, pos, chr); + if(commentCount !== 0){ + this.state.commenting = true; + continue; + } + const delimiterLength = this.__isDelimiter(buf, pos, chr); + if(delimiterLength !== 0){ this.info.bytes = this.state.bufBytesStart + pos; const errField = this.__onField(); if(errField !== undefined) return errField; - this.info.bytes = this.state.bufBytesStart + pos + recordDelimiterLength; - const errRecord = this.__onRecord(); - if(errRecord !== undefined) return errRecord; - if(to !== -1 && this.info.records >= to){ - this.state.stop = true; - this.push(null); - return; - } + pos += delimiterLength - 1; + continue; } - this.state.commenting = false; - pos += recordDelimiterLength - 1; - continue; - } - if(this.state.commenting){ - continue; } - const commentCount = comment === null ? 0 : this.__compareBytes(comment, buf, pos, chr); - if(commentCount !== 0){ - this.state.commenting = true; - continue; - } - const delimiterLength = this.__isDelimiter(buf, pos, chr); - if(delimiterLength !== 0){ - this.info.bytes = this.state.bufBytesStart + pos; - const errField = this.__onField(); - if(errField !== undefined) return errField; - pos += delimiterLength - 1; - continue; + } + if(this.state.commenting === false){ + if(max_record_size !== 0 && this.state.record_length + this.state.field.length > max_record_size){ + const err = this.__error( + new CsvError('CSV_MAX_RECORD_SIZE', [ + 'Max Record Size:', + 'record exceed the maximum number of tolerated bytes', + `of ${max_record_size}`, + `at line ${this.info.lines}`, + ], this.options, this.__infoField()) + ); + if(err !== undefined) return err; } } - } - if(this.state.commenting === false){ - if(max_record_size !== 0 && this.state.record_length + this.state.field.length > max_record_size){ + const lappend = ltrim === false || this.state.quoting === true || this.state.field.length !== 0 || !this.__isCharTrimable(chr); + // rtrim in non quoting is handle in __onField + const rappend = rtrim === false || this.state.wasQuoting === false; + if(lappend === true && rappend === true){ + this.state.field.append(chr); + }else if(rtrim === true && !this.__isCharTrimable(chr)){ const err = this.__error( - new CsvError('CSV_MAX_RECORD_SIZE', [ - 'Max Record Size:', - 'record exceed the maximum number of tolerated bytes', - `of ${max_record_size}`, + new CsvError('CSV_NON_TRIMABLE_CHAR_AFTER_CLOSING_QUOTE', [ + 'Invalid Closing Quote:', + 'found non trimable byte after quote', `at line ${this.info.lines}`, ], this.options, this.__infoField()) ); if(err !== undefined) return err; } } - const lappend = ltrim === false || this.state.quoting === true || this.state.field.length !== 0 || !this.__isCharTrimable(chr); - // rtrim in non quoting is handle in __onField - const rappend = rtrim === false || this.state.wasQuoting === false; - if(lappend === true && rappend === true){ - this.state.field.append(chr); - }else if(rtrim === true && !this.__isCharTrimable(chr)){ - const err = this.__error( - new CsvError('CSV_NON_TRIMABLE_CHAR_AFTER_CLOSING_QUOTE', [ - 'Invalid Closing Quote:', - 'found non trimable byte after quote', - `at line ${this.info.lines}`, - ], this.options, this.__infoField()) - ); - if(err !== undefined) return err; - } - } - if(end === true){ - // Ensure we are not ending in a quoting state - if(this.state.quoting === true){ - const err = this.__error( - new CsvError('CSV_QUOTE_NOT_CLOSED', [ - 'Quote Not Closed:', - `the parsing is finished with an opening quote at line ${this.info.lines}`, - ], this.options, this.__infoField()) - ); - if(err !== undefined) return err; + if(end === true){ + // Ensure we are not ending in a quoting state + if(this.state.quoting === true){ + const err = this.__error( + new CsvError('CSV_QUOTE_NOT_CLOSED', [ + 'Quote Not Closed:', + `the parsing is finished with an opening quote at line ${this.info.lines}`, + ], this.options, this.__infoField()) + ); + if(err !== undefined) return err; + }else { + // Skip last line if it has no characters + if(this.state.wasQuoting === true || this.state.record.length !== 0 || this.state.field.length !== 0){ + this.info.bytes = this.state.bufBytesStart + pos; + const errField = this.__onField(); + if(errField !== undefined) return errField; + const errRecord = this.__onRecord(push); + if(errRecord !== undefined) return errRecord; + }else if(this.state.wasRowDelimiter === true){ + this.info.empty_lines++; + }else if(this.state.commenting === true){ + this.info.comment_lines++; + } + } }else { - // Skip last line if it has no characters - if(this.state.wasQuoting === true || this.state.record.length !== 0 || this.state.field.length !== 0){ - this.info.bytes = this.state.bufBytesStart + pos; - const errField = this.__onField(); - if(errField !== undefined) return errField; - const errRecord = this.__onRecord(); - if(errRecord !== undefined) return errRecord; - }else if(this.state.wasRowDelimiter === true){ - this.info.empty_lines++; - }else if(this.state.commenting === true){ - this.info.comment_lines++; + this.state.bufBytesStart += pos; + this.state.previousBuf = buf.slice(pos); + } + if(this.state.wasRowDelimiter === true){ + this.info.lines++; + this.state.wasRowDelimiter = false; + } + }, + __onRecord: function(push){ + const {columns, group_columns_by_name, encoding, info, from, relax_column_count, relax_column_count_less, relax_column_count_more, raw, skip_records_with_empty_values} = this.options; + const {enabled, record} = this.state; + if(enabled === false){ + return this.__resetRecord(); + } + // Convert the first line into column names + const recordLength = record.length; + if(columns === true){ + if(skip_records_with_empty_values === true && isRecordEmpty(record)){ + this.__resetRecord(); + return; + } + return this.__firstLineToColumns(record); + } + if(columns === false && this.info.records === 0){ + this.state.expectedRecordLength = recordLength; + } + if(recordLength !== this.state.expectedRecordLength){ + const err = columns === false ? + new CsvError('CSV_RECORD_INCONSISTENT_FIELDS_LENGTH', [ + 'Invalid Record Length:', + `expect ${this.state.expectedRecordLength},`, + `got ${recordLength} on line ${this.info.lines}`, + ], this.options, this.__infoField(), { + record: record, + }) + : + new CsvError('CSV_RECORD_INCONSISTENT_COLUMNS', [ + 'Invalid Record Length:', + `columns length is ${columns.length},`, // rename columns + `got ${recordLength} on line ${this.info.lines}`, + ], this.options, this.__infoField(), { + record: record, + }); + if(relax_column_count === true || + (relax_column_count_less === true && recordLength < this.state.expectedRecordLength) || + (relax_column_count_more === true && recordLength > this.state.expectedRecordLength)){ + this.info.invalid_field_length++; + this.state.error = err; + // Error is undefined with skip_records_with_error + }else { + const finalErr = this.__error(err); + if(finalErr) return finalErr; } } - }else { - this.state.bufBytesStart += pos; - this.state.previousBuf = buf.slice(pos); - } - if(this.state.wasRowDelimiter === true){ - this.info.lines++; - this.state.wasRowDelimiter = false; - } - } - __onRecord(){ - const {columns, group_columns_by_name, encoding, info, from, relax_column_count, relax_column_count_less, relax_column_count_more, raw, skip_records_with_empty_values} = this.options; - const {enabled, record} = this.state; - if(enabled === false){ - return this.__resetRecord(); - } - // Convert the first line into column names - const recordLength = record.length; - if(columns === true){ if(skip_records_with_empty_values === true && isRecordEmpty(record)){ this.__resetRecord(); return; } - return this.__firstLineToColumns(record); - } - if(columns === false && this.info.records === 0){ - this.state.expectedRecordLength = recordLength; - } - if(recordLength !== this.state.expectedRecordLength){ - const err = columns === false ? - new CsvError('CSV_RECORD_INCONSISTENT_FIELDS_LENGTH', [ - 'Invalid Record Length:', - `expect ${this.state.expectedRecordLength},`, - `got ${recordLength} on line ${this.info.lines}`, - ], this.options, this.__infoField(), { - record: record, - }) - : - new CsvError('CSV_RECORD_INCONSISTENT_COLUMNS', [ - 'Invalid Record Length:', - `columns length is ${columns.length},`, // rename columns - `got ${recordLength} on line ${this.info.lines}`, - ], this.options, this.__infoField(), { - record: record, - }); - if(relax_column_count === true || - (relax_column_count_less === true && recordLength < this.state.expectedRecordLength) || - (relax_column_count_more === true && recordLength > this.state.expectedRecordLength)){ - this.info.invalid_field_length++; - this.state.error = err; - // Error is undefined with skip_records_with_error - }else { - const finalErr = this.__error(err); - if(finalErr) return finalErr; + if(this.state.recordHasError === true){ + this.__resetRecord(); + this.state.recordHasError = false; + return; } - } - if(skip_records_with_empty_values === true && isRecordEmpty(record)){ - this.__resetRecord(); - return; - } - if(this.state.recordHasError === true){ - this.__resetRecord(); - this.state.recordHasError = false; - return; - } - this.info.records++; - if(from === 1 || this.info.records >= from){ - const {objname} = this.options; - // With columns, records are object - if(columns !== false){ - const obj = {}; - // Transform record array to an object - for(let i = 0, l = record.length; i < l; i++){ - if(columns[i] === undefined || columns[i].disabled) continue; - // Turn duplicate columns into an array - if (group_columns_by_name === true && obj[columns[i].name] !== undefined) { - if (Array.isArray(obj[columns[i].name])) { - obj[columns[i].name] = obj[columns[i].name].concat(record[i]); + this.info.records++; + if(from === 1 || this.info.records >= from){ + const {objname} = this.options; + // With columns, records are object + if(columns !== false){ + const obj = {}; + // Transform record array to an object + for(let i = 0, l = record.length; i < l; i++){ + if(columns[i] === undefined || columns[i].disabled) continue; + // Turn duplicate columns into an array + if (group_columns_by_name === true && obj[columns[i].name] !== undefined) { + if (Array.isArray(obj[columns[i].name])) { + obj[columns[i].name] = obj[columns[i].name].concat(record[i]); + } else { + obj[columns[i].name] = [obj[columns[i].name], record[i]]; + } } else { - obj[columns[i].name] = [obj[columns[i].name], record[i]]; + obj[columns[i].name] = record[i]; } - } else { - obj[columns[i].name] = record[i]; - } - } - // Without objname (default) - if(raw === true || info === true){ - const extRecord = Object.assign( - {record: obj}, - (raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}), - (info === true ? {info: this.__infoRecord()}: {}) - ); - const err = this.__push( - objname === undefined ? extRecord : [obj[objname], extRecord] - ); - if(err){ - return err; - } - }else { - const err = this.__push( - objname === undefined ? obj : [obj[objname], obj] - ); - if(err){ - return err; } - } - // Without columns, records are array - }else { - if(raw === true || info === true){ - const extRecord = Object.assign( - {record: record}, - raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}, - info === true ? {info: this.__infoRecord()}: {} - ); - const err = this.__push( - objname === undefined ? extRecord : [record[objname], extRecord] - ); - if(err){ - return err; + // Without objname (default) + if(raw === true || info === true){ + const extRecord = Object.assign( + {record: obj}, + (raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}), + (info === true ? {info: this.__infoRecord()}: {}) + ); + const err = this.__push( + objname === undefined ? extRecord : [obj[objname], extRecord] + , push); + if(err){ + return err; + } + }else { + const err = this.__push( + objname === undefined ? obj : [obj[objname], obj] + , push); + if(err){ + return err; + } } + // Without columns, records are array }else { - const err = this.__push( - objname === undefined ? record : [record[objname], record] - ); - if(err){ - return err; + if(raw === true || info === true){ + const extRecord = Object.assign( + {record: record}, + raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}, + info === true ? {info: this.__infoRecord()}: {} + ); + const err = this.__push( + objname === undefined ? extRecord : [record[objname], extRecord] + , push); + if(err){ + return err; + } + }else { + const err = this.__push( + objname === undefined ? record : [record[objname], record] + , push); + if(err){ + return err; + } } } } - } - this.__resetRecord(); - } - __firstLineToColumns(record){ - const {firstLineToHeaders} = this.state; - try{ - const headers = firstLineToHeaders === undefined ? record : firstLineToHeaders.call(null, record); - if(!Array.isArray(headers)){ - return this.__error( - new CsvError('CSV_INVALID_COLUMN_MAPPING', [ - 'Invalid Column Mapping:', - 'expect an array from column function,', - `got ${JSON.stringify(headers)}` - ], this.options, this.__infoField(), { - headers: headers, - }) - ); - } - const normalizedHeaders = normalizeColumnsArray(headers); - this.state.expectedRecordLength = normalizedHeaders.length; - this.options.columns = normalizedHeaders; this.__resetRecord(); - return; - }catch(err){ - return err; - } - } - __resetRecord(){ - if(this.options.raw === true){ - this.state.rawBuffer.reset(); - } - this.state.error = undefined; - this.state.record = []; - this.state.record_length = 0; - } - __onField(){ - const {cast, encoding, rtrim, max_record_size} = this.options; - const {enabled, wasQuoting} = this.state; - // Short circuit for the from_line options - if(enabled === false){ - return this.__resetField(); - } - let field = this.state.field.toString(encoding); - if(rtrim === true && wasQuoting === false){ - field = field.trimRight(); - } - if(cast === true){ - const [err, f] = this.__cast(field); - if(err !== undefined) return err; - field = f; - } - this.state.record.push(field); - // Increment record length if record size must not exceed a limit - if(max_record_size !== 0 && typeof field === 'string'){ - this.state.record_length += field.length; - } - this.__resetField(); - } - __resetField(){ - this.state.field.reset(); - this.state.wasQuoting = false; - } - __push(record){ - const {on_record} = this.options; - if(on_record !== undefined){ - const info = this.__infoRecord(); + }, + __firstLineToColumns: function(record){ + const {firstLineToHeaders} = this.state; try{ - record = on_record.call(null, record, info); + const headers = firstLineToHeaders === undefined ? record : firstLineToHeaders.call(null, record); + if(!Array.isArray(headers)){ + return this.__error( + new CsvError('CSV_INVALID_COLUMN_MAPPING', [ + 'Invalid Column Mapping:', + 'expect an array from column function,', + `got ${JSON.stringify(headers)}` + ], this.options, this.__infoField(), { + headers: headers, + }) + ); + } + const normalizedHeaders = normalize_columns_array(headers); + this.state.expectedRecordLength = normalizedHeaders.length; + this.options.columns = normalizedHeaders; + this.__resetRecord(); + return; }catch(err){ return err; } - if(record === undefined || record === null){ return; } - } - this.push(record); - } - // Return a tuple with the error and the casted value - __cast(field){ - const {columns, relax_column_count} = this.options; - const isColumns = Array.isArray(columns); - // Dont loose time calling cast - // because the final record is an object - // and this field can't be associated to a key present in columns - if(isColumns === true && relax_column_count && this.options.columns.length <= this.state.record.length){ - return [undefined, undefined]; - } - if(this.state.castField !== null){ - try{ + }, + __resetRecord: function(){ + if(this.options.raw === true){ + this.state.rawBuffer.reset(); + } + this.state.error = undefined; + this.state.record = []; + this.state.record_length = 0; + }, + __onField: function(){ + const {cast, encoding, rtrim, max_record_size} = this.options; + const {enabled, wasQuoting} = this.state; + // Short circuit for the from_line options + if(enabled === false){ + return this.__resetField(); + } + let field = this.state.field.toString(encoding); + if(rtrim === true && wasQuoting === false){ + field = field.trimRight(); + } + if(cast === true){ + const [err, f] = this.__cast(field); + if(err !== undefined) return err; + field = f; + } + this.state.record.push(field); + // Increment record length if record size must not exceed a limit + if(max_record_size !== 0 && typeof field === 'string'){ + this.state.record_length += field.length; + } + this.__resetField(); + }, + __resetField: function(){ + this.state.field.reset(); + this.state.wasQuoting = false; + }, + __push: function(record, push){ + const {on_record} = this.options; + if(on_record !== undefined){ + const info = this.__infoRecord(); + try{ + record = on_record.call(null, record, info); + }catch(err){ + return err; + } + if(record === undefined || record === null){ return; } + } + push(record); + }, + // Return a tuple with the error and the casted value + __cast: function(field){ + const {columns, relax_column_count} = this.options; + const isColumns = Array.isArray(columns); + // Dont loose time calling cast + // because the final record is an object + // and this field can't be associated to a key present in columns + if(isColumns === true && relax_column_count && this.options.columns.length <= this.state.record.length){ + return [undefined, undefined]; + } + if(this.state.castField !== null){ + try{ + const info = this.__infoField(); + return [undefined, this.state.castField.call(null, field, info)]; + }catch(err){ + return [err]; + } + } + if(this.__isFloat(field)){ + return [undefined, parseFloat(field)]; + }else if(this.options.cast_date !== false){ const info = this.__infoField(); - return [undefined, this.state.castField.call(null, field, info)]; - }catch(err){ - return [err]; + return [undefined, this.options.cast_date.call(null, field, info)]; + } + return [undefined, field]; + }, + // Helper to test if a character is a space or a line delimiter + __isCharTrimable: function(chr){ + return chr === space || chr === tab || chr === cr || chr === nl || chr === np; + }, + // Keep it in case we implement the `cast_int` option + // __isInt(value){ + // // return Number.isInteger(parseInt(value)) + // // return !isNaN( parseInt( obj ) ); + // return /^(\-|\+)?[1-9][0-9]*$/.test(value) + // } + __isFloat: function(value){ + return (value - parseFloat(value) + 1) >= 0; // Borrowed from jquery + }, + __compareBytes: function(sourceBuf, targetBuf, targetPos, firstByte){ + if(sourceBuf[0] !== firstByte) return 0; + const sourceLength = sourceBuf.length; + for(let i = 1; i < sourceLength; i++){ + if(sourceBuf[i] !== targetBuf[targetPos+i]) return 0; + } + return sourceLength; + }, + __isDelimiter: function(buf, pos, chr){ + const {delimiter, ignore_last_delimiters} = this.options; + if(ignore_last_delimiters === true && this.state.record.length === this.options.columns.length - 1){ + return 0; + }else if(ignore_last_delimiters !== false && typeof ignore_last_delimiters === 'number' && this.state.record.length === ignore_last_delimiters - 1){ + return 0; + } + loop1: for(let i = 0; i < delimiter.length; i++){ + const del = delimiter[i]; + if(del[0] === chr){ + for(let j = 1; j < del.length; j++){ + if(del[j] !== buf[pos+j]) continue loop1; + } + return del.length; + } } - } - if(this.__isFloat(field)){ - return [undefined, parseFloat(field)]; - }else if(this.options.cast_date !== false){ - const info = this.__infoField(); - return [undefined, this.options.cast_date.call(null, field, info)]; - } - return [undefined, field]; - } - // Helper to test if a character is a space or a line delimiter - __isCharTrimable(chr){ - return chr === space || chr === tab || chr === cr || chr === nl || chr === np; - } - // Keep it in case we implement the `cast_int` option - // __isInt(value){ - // // return Number.isInteger(parseInt(value)) - // // return !isNaN( parseInt( obj ) ); - // return /^(\-|\+)?[1-9][0-9]*$/.test(value) - // } - __isFloat(value){ - return (value - parseFloat(value) + 1) >= 0; // Borrowed from jquery - } - __compareBytes(sourceBuf, targetBuf, targetPos, firstByte){ - if(sourceBuf[0] !== firstByte) return 0; - const sourceLength = sourceBuf.length; - for(let i = 1; i < sourceLength; i++){ - if(sourceBuf[i] !== targetBuf[targetPos+i]) return 0; - } - return sourceLength; - } - __needMoreData(i, bufLen, end){ - if(end) return false; - const {quote} = this.options; - const {quoting, needMoreDataSize, recordDelimiterMaxLength} = this.state; - const numOfCharLeft = bufLen - i - 1; - const requiredLength = Math.max( - needMoreDataSize, - // Skip if the remaining buffer smaller than record delimiter - recordDelimiterMaxLength, - // Skip if the remaining buffer can be record delimiter following the closing quote - // 1 is for quote.length - quoting ? (quote.length + recordDelimiterMaxLength) : 0, - ); - return numOfCharLeft < requiredLength; - } - __isDelimiter(buf, pos, chr){ - const {delimiter, ignore_last_delimiters} = this.options; - if(ignore_last_delimiters === true && this.state.record.length === this.options.columns.length - 1){ - return 0; - }else if(ignore_last_delimiters !== false && typeof ignore_last_delimiters === 'number' && this.state.record.length === ignore_last_delimiters - 1){ return 0; - } - loop1: for(let i = 0; i < delimiter.length; i++){ - const del = delimiter[i]; - if(del[0] === chr){ - for(let j = 1; j < del.length; j++){ - if(del[j] !== buf[pos+j]) continue loop1; + }, + __isRecordDelimiter: function(chr, buf, pos){ + const {record_delimiter} = this.options; + const recordDelimiterLength = record_delimiter.length; + loop1: for(let i = 0; i < recordDelimiterLength; i++){ + const rd = record_delimiter[i]; + const rdLength = rd.length; + if(rd[0] !== chr){ + continue; } - return del.length; - } - } - return 0; - } - __isRecordDelimiter(chr, buf, pos){ - const {record_delimiter} = this.options; - const recordDelimiterLength = record_delimiter.length; - loop1: for(let i = 0; i < recordDelimiterLength; i++){ - const rd = record_delimiter[i]; - const rdLength = rd.length; - if(rd[0] !== chr){ - continue; - } - for(let j = 1; j < rdLength; j++){ - if(rd[j] !== buf[pos+j]){ - continue loop1; + for(let j = 1; j < rdLength; j++){ + if(rd[j] !== buf[pos+j]){ + continue loop1; + } } + return rd.length; } - return rd.length; - } - return 0; - } - __isEscape(buf, pos, chr){ - const {escape} = this.options; - if(escape === null) return false; - const l = escape.length; - if(escape[0] === chr){ + return 0; + }, + __isEscape: function(buf, pos, chr){ + const {escape} = this.options; + if(escape === null) return false; + const l = escape.length; + if(escape[0] === chr){ + for(let i = 0; i < l; i++){ + if(escape[i] !== buf[pos+i]){ + return false; + } + } + return true; + } + return false; + }, + __isQuote: function(buf, pos){ + const {quote} = this.options; + if(quote === null) return false; + const l = quote.length; for(let i = 0; i < l; i++){ - if(escape[i] !== buf[pos+i]){ + if(quote[i] !== buf[pos+i]){ return false; } } return true; - } - return false; - } - __isQuote(buf, pos){ - const {quote} = this.options; - if(quote === null) return false; - const l = quote.length; - for(let i = 0; i < l; i++){ - if(quote[i] !== buf[pos+i]){ - return false; - } - } - return true; - } - __autoDiscoverRecordDelimiter(buf, pos){ - const {encoding} = this.options; - const chr = buf[pos]; - if(chr === cr){ - if(buf[pos+1] === nl){ - this.options.record_delimiter.push(Buffer.from('\r\n', encoding)); - this.state.recordDelimiterMaxLength = 2; - return 2; - }else { - this.options.record_delimiter.push(Buffer.from('\r', encoding)); + }, + __autoDiscoverRecordDelimiter: function(buf, pos){ + const {encoding} = this.options; + const chr = buf[pos]; + if(chr === cr){ + if(buf[pos+1] === nl){ + this.options.record_delimiter.push(Buffer.from('\r\n', encoding)); + this.state.recordDelimiterMaxLength = 2; + return 2; + }else { + this.options.record_delimiter.push(Buffer.from('\r', encoding)); + this.state.recordDelimiterMaxLength = 1; + return 1; + } + }else if(chr === nl){ + this.options.record_delimiter.push(Buffer.from('\n', encoding)); this.state.recordDelimiterMaxLength = 1; return 1; } - }else if(chr === nl){ - this.options.record_delimiter.push(Buffer.from('\n', encoding)); - this.state.recordDelimiterMaxLength = 1; - return 1; - } - return 0; - } - __error(msg){ - const {encoding, raw, skip_records_with_error} = this.options; - const err = typeof msg === 'string' ? new Error(msg) : msg; - if(skip_records_with_error){ - this.state.recordHasError = true; - this.emit('skip', err, raw ? this.state.rawBuffer.toString(encoding) : undefined); - return undefined; - }else { - return err; + return 0; + }, + __error: function(msg){ + const {encoding, raw, skip_records_with_error} = this.options; + const err = typeof msg === 'string' ? new Error(msg) : msg; + if(skip_records_with_error){ + this.state.recordHasError = true; + if(this.options.on_skip !== undefined){ + this.options.on_skip(err, raw ? this.state.rawBuffer.toString(encoding) : undefined); + } + // this.emit('skip', err, raw ? this.state.rawBuffer.toString(encoding) : undefined); + return undefined; + }else { + return err; + } + }, + __infoDataSet: function(){ + return { + ...this.info, + columns: this.options.columns + }; + }, + __infoRecord: function(){ + const {columns, raw, encoding} = this.options; + return { + ...this.__infoDataSet(), + error: this.state.error, + header: columns === true, + index: this.state.record.length, + raw: raw ? this.state.rawBuffer.toString(encoding) : undefined + }; + }, + __infoField: function(){ + const {columns} = this.options; + const isColumns = Array.isArray(columns); + return { + ...this.__infoRecord(), + column: isColumns === true ? + (columns.length > this.state.record.length ? + columns[this.state.record.length].name : + null + ) : + this.state.record.length, + quoting: this.state.wasQuoting, + }; } - } - __infoDataSet(){ - return { - ...this.info, - columns: this.options.columns - }; - } - __infoRecord(){ - const {columns, raw, encoding} = this.options; - return { - ...this.__infoDataSet(), - error: this.state.error, - header: columns === true, - index: this.state.record.length, - raw: raw ? this.state.rawBuffer.toString(encoding) : undefined - }; - } - __infoField(){ - const {columns} = this.options; - const isColumns = Array.isArray(columns); - return { - ...this.__infoRecord(), - column: isColumns === true ? - (columns.length > this.state.record.length ? - columns[this.state.record.length].name : - null - ) : - this.state.record.length, - quoting: this.state.wasQuoting, - }; - } -} + }; +}; -const parse = function(data, options={}){ +const parse = function(data, opts={}){ if(typeof data === 'string'){ data = Buffer.from(data); } - const records = options && options.objname ? {} : []; - const parser = new Parser(options); - parser.push = function(record){ - if(record === null){ - return; - } - if(options.objname === undefined) + const records = opts && opts.objname ? {} : []; + const parser = transform(opts); + const push = (record) => { + if(parser.options.objname === undefined) records.push(record); else { records[record[0]] = record[1]; } }; - const err1 = parser.__parse(data, false); + const close = () => {}; + const err1 = parser.parse(data, false, push, close); if(err1 !== undefined) throw err1; - const err2 = parser.__parse(undefined, true); + const err2 = parser.parse(undefined, true, push, close); if(err2 !== undefined) throw err2; return records; }; diff --git a/packages/csv-parse/dist/iife/index.js b/packages/csv-parse/dist/iife/index.js index ce9849bee..e0e38d690 100644 --- a/packages/csv-parse/dist/iife/index.js +++ b/packages/csv-parse/dist/iife/index.js @@ -2630,7 +2630,7 @@ var csv_parse = (function (exports) { } }); for (var x = args[i]; i < len; x = args[++i]) { - if (isNull(x) || !isObject$1(x)) { + if (isNull(x) || !isObject(x)) { str += ' ' + x; } else { str += ' ' + inspect(x); @@ -3038,19 +3038,19 @@ var csv_parse = (function (exports) { } function isRegExp(re) { - return isObject$1(re) && objectToString(re) === '[object RegExp]'; + return isObject(re) && objectToString(re) === '[object RegExp]'; } - function isObject$1(arg) { + function isObject(arg) { return typeof arg === 'object' && arg !== null; } function isDate(d) { - return isObject$1(d) && objectToString(d) === '[object Date]'; + return isObject(d) && objectToString(d) === '[object Date]'; } function isError(e) { - return isObject$1(e) && + return isObject(e) && (objectToString(e) === '[object Error]' || e instanceof Error); } @@ -3064,7 +3064,7 @@ var csv_parse = (function (exports) { function _extend(origin, add) { // Don't do anything if add isn't an object - if (!add || !isObject$1(add)) return origin; + if (!add || !isObject(add)) return origin; var keys = Object.keys(add); var i = keys.length; @@ -4969,6 +4969,55 @@ var csv_parse = (function (exports) { return dest; }; + const is_object = function(obj){ + return (typeof obj === 'object' && obj !== null && !Array.isArray(obj)); + }; + + class CsvError extends Error { + constructor(code, message, options, ...contexts) { + if(Array.isArray(message)) message = message.join(' '); + super(message); + if(Error.captureStackTrace !== undefined){ + Error.captureStackTrace(this, CsvError); + } + this.code = code; + for(const context of contexts){ + for(const key in context){ + const value = context[key]; + this[key] = isBuffer(value) ? value.toString(options.encoding) : value == null ? value : JSON.parse(JSON.stringify(value)); + } + } + } + } + + const normalize_columns_array = function(columns){ + const normalizedColumns = []; + for(let i = 0, l = columns.length; i < l; i++){ + const column = columns[i]; + if(column === undefined || column === null || column === false){ + normalizedColumns[i] = { disabled: true }; + }else if(typeof column === 'string'){ + normalizedColumns[i] = { name: column }; + }else if(is_object(column)){ + if(typeof column.name !== 'string'){ + throw new CsvError('CSV_OPTION_COLUMNS_MISSING_NAME', [ + 'Option columns missing name:', + `property "name" is required at position ${i}`, + 'when column is an object literal' + ]); + } + normalizedColumns[i] = column; + }else { + throw new CsvError('CSV_INVALID_COLUMN_DEFINITION', [ + 'Invalid column definition:', + 'expect a string or a literal object,', + `got ${JSON.stringify(column)} at position ${i}` + ]); + } + } + return normalizedColumns; + }; + class ResizeableBuffer{ constructor(size=100){ this.size = size; @@ -5031,1210 +5080,1192 @@ var csv_parse = (function (exports) { } } - // white space characters - // https://en.wikipedia.org/wiki/Whitespace_character - // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions/Character_Classes#Types - // \f\n\r\t\v\u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff - const tab = 9; - const nl = 10; // \n, 0x0A in hexadecimal, 10 in decimal - const np = 12; - const cr = 13; // \r, 0x0D in hexadécimal, 13 in decimal - const space = 32; - const boms = { - // Note, the following are equals: - // Buffer.from("\ufeff") - // Buffer.from([239, 187, 191]) - // Buffer.from('EFBBBF', 'hex') - 'utf8': Buffer.from([239, 187, 191]), - // Note, the following are equals: - // Buffer.from "\ufeff", 'utf16le - // Buffer.from([255, 254]) - 'utf16le': Buffer.from([255, 254]) + const init_state = function(options){ + return { + bomSkipped: false, + bufBytesStart: 0, + castField: options.cast_function, + commenting: false, + // Current error encountered by a record + error: undefined, + enabled: options.from_line === 1, + escaping: false, + escapeIsQuote: isBuffer(options.escape) && isBuffer(options.quote) && Buffer.compare(options.escape, options.quote) === 0, + // columns can be `false`, `true`, `Array` + expectedRecordLength: Array.isArray(options.columns) ? options.columns.length : undefined, + field: new ResizeableBuffer(20), + firstLineToHeaders: options.cast_first_line_to_header, + needMoreDataSize: Math.max( + // Skip if the remaining buffer smaller than comment + options.comment !== null ? options.comment.length : 0, + // Skip if the remaining buffer can be delimiter + ...options.delimiter.map((delimiter) => delimiter.length), + // Skip if the remaining buffer can be escape sequence + options.quote !== null ? options.quote.length : 0, + ), + previousBuf: undefined, + quoting: false, + stop: false, + rawBuffer: new ResizeableBuffer(100), + record: [], + recordHasError: false, + record_length: 0, + recordDelimiterMaxLength: options.record_delimiter.length === 0 ? 2 : Math.max(...options.record_delimiter.map((v) => v.length)), + trimChars: [Buffer.from(' ', options.encoding)[0], Buffer.from('\t', options.encoding)[0]], + wasQuoting: false, + wasRowDelimiter: false + }; }; - class CsvError extends Error { - constructor(code, message, options, ...contexts) { - if(Array.isArray(message)) message = message.join(' '); - super(message); - if(Error.captureStackTrace !== undefined){ - Error.captureStackTrace(this, CsvError); - } - this.code = code; - for(const context of contexts){ - for(const key in context){ - const value = context[key]; - this[key] = isBuffer(value) ? value.toString(options.encoding) : value == null ? value : JSON.parse(JSON.stringify(value)); - } - } - } - } - const underscore = function(str){ return str.replace(/([A-Z])/g, function(_, match){ return '_' + match.toLowerCase(); }); }; - const isObject = function(obj){ - return (typeof obj === 'object' && obj !== null && !Array.isArray(obj)); - }; - - const isRecordEmpty = function(record){ - return record.every((field) => field == null || field.toString && field.toString().trim() === ''); - }; - - const normalizeColumnsArray = function(columns){ - const normalizedColumns = []; - for(let i = 0, l = columns.length; i < l; i++){ - const column = columns[i]; - if(column === undefined || column === null || column === false){ - normalizedColumns[i] = { disabled: true }; - }else if(typeof column === 'string'){ - normalizedColumns[i] = { name: column }; - }else if(isObject(column)){ - if(typeof column.name !== 'string'){ - throw new CsvError('CSV_OPTION_COLUMNS_MISSING_NAME', [ - 'Option columns missing name:', - `property "name" is required at position ${i}`, - 'when column is an object literal' - ]); - } - normalizedColumns[i] = column; - }else { - throw new CsvError('CSV_INVALID_COLUMN_DEFINITION', [ - 'Invalid column definition:', - 'expect a string or a literal object,', - `got ${JSON.stringify(column)} at position ${i}` - ]); - } - } - return normalizedColumns; - }; - - class Parser extends Transform { - constructor(opts = {}){ - super({...{readableObjectMode: true}, ...opts, encoding: null}); - this.__originalOptions = opts; - this.__normalizeOptions(opts); - } - __normalizeOptions(opts){ - const options = {}; - // Merge with user options - for(const opt in opts){ - options[underscore(opt)] = opts[opt]; - } - // Normalize option `encoding` - // Note: defined first because other options depends on it - // to convert chars/strings into buffers. - if(options.encoding === undefined || options.encoding === true){ - options.encoding = 'utf8'; - }else if(options.encoding === null || options.encoding === false){ - options.encoding = null; - }else if(typeof options.encoding !== 'string' && options.encoding !== null){ - throw new CsvError('CSV_INVALID_OPTION_ENCODING', [ - 'Invalid option encoding:', - 'encoding must be a string or null to return a buffer,', - `got ${JSON.stringify(options.encoding)}` - ], options); - } - // Normalize option `bom` - if(options.bom === undefined || options.bom === null || options.bom === false){ - options.bom = false; - }else if(options.bom !== true){ - throw new CsvError('CSV_INVALID_OPTION_BOM', [ - 'Invalid option bom:', 'bom must be true,', - `got ${JSON.stringify(options.bom)}` - ], options); - } - // Normalize option `cast` - let fnCastField = null; - if(options.cast === undefined || options.cast === null || options.cast === false || options.cast === ''){ - options.cast = undefined; - }else if(typeof options.cast === 'function'){ - fnCastField = options.cast; - options.cast = true; - }else if(options.cast !== true){ - throw new CsvError('CSV_INVALID_OPTION_CAST', [ - 'Invalid option cast:', 'cast must be true or a function,', - `got ${JSON.stringify(options.cast)}` - ], options); - } - // Normalize option `cast_date` - if(options.cast_date === undefined || options.cast_date === null || options.cast_date === false || options.cast_date === ''){ - options.cast_date = false; - }else if(options.cast_date === true){ - options.cast_date = function(value){ - const date = Date.parse(value); - return !isNaN(date) ? new Date(date) : value; - }; - }else { - throw new CsvError('CSV_INVALID_OPTION_CAST_DATE', [ - 'Invalid option cast_date:', 'cast_date must be true or a function,', - `got ${JSON.stringify(options.cast_date)}` + const normalize_options = function(opts){ + const options = {}; + // Merge with user options + for(const opt in opts){ + options[underscore(opt)] = opts[opt]; + } + // Normalize option `encoding` + // Note: defined first because other options depends on it + // to convert chars/strings into buffers. + if(options.encoding === undefined || options.encoding === true){ + options.encoding = 'utf8'; + }else if(options.encoding === null || options.encoding === false){ + options.encoding = null; + }else if(typeof options.encoding !== 'string' && options.encoding !== null){ + throw new CsvError('CSV_INVALID_OPTION_ENCODING', [ + 'Invalid option encoding:', + 'encoding must be a string or null to return a buffer,', + `got ${JSON.stringify(options.encoding)}` + ], options); + } + // Normalize option `bom` + if(options.bom === undefined || options.bom === null || options.bom === false){ + options.bom = false; + }else if(options.bom !== true){ + throw new CsvError('CSV_INVALID_OPTION_BOM', [ + 'Invalid option bom:', 'bom must be true,', + `got ${JSON.stringify(options.bom)}` + ], options); + } + // Normalize option `cast` + options.cast_function = null; + if(options.cast === undefined || options.cast === null || options.cast === false || options.cast === ''){ + options.cast = undefined; + }else if(typeof options.cast === 'function'){ + options.cast_function = options.cast; + options.cast = true; + }else if(options.cast !== true){ + throw new CsvError('CSV_INVALID_OPTION_CAST', [ + 'Invalid option cast:', 'cast must be true or a function,', + `got ${JSON.stringify(options.cast)}` + ], options); + } + // Normalize option `cast_date` + if(options.cast_date === undefined || options.cast_date === null || options.cast_date === false || options.cast_date === ''){ + options.cast_date = false; + }else if(options.cast_date === true){ + options.cast_date = function(value){ + const date = Date.parse(value); + return !isNaN(date) ? new Date(date) : value; + }; + }else { + throw new CsvError('CSV_INVALID_OPTION_CAST_DATE', [ + 'Invalid option cast_date:', 'cast_date must be true or a function,', + `got ${JSON.stringify(options.cast_date)}` + ], options); + } + // Normalize option `columns` + options.cast_first_line_to_header = null; + if(options.columns === true){ + // Fields in the first line are converted as-is to columns + options.cast_first_line_to_header = undefined; + }else if(typeof options.columns === 'function'){ + options.cast_first_line_to_header = options.columns; + options.columns = true; + }else if(Array.isArray(options.columns)){ + options.columns = normalize_columns_array(options.columns); + }else if(options.columns === undefined || options.columns === null || options.columns === false){ + options.columns = false; + }else { + throw new CsvError('CSV_INVALID_OPTION_COLUMNS', [ + 'Invalid option columns:', + 'expect an array, a function or true,', + `got ${JSON.stringify(options.columns)}` + ], options); + } + // Normalize option `group_columns_by_name` + if(options.group_columns_by_name === undefined || options.group_columns_by_name === null || options.group_columns_by_name === false){ + options.group_columns_by_name = false; + }else if(options.group_columns_by_name !== true){ + throw new CsvError('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ + 'Invalid option group_columns_by_name:', + 'expect an boolean,', + `got ${JSON.stringify(options.group_columns_by_name)}` + ], options); + }else if(options.columns === false){ + throw new CsvError('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ + 'Invalid option group_columns_by_name:', + 'the `columns` mode must be activated.' + ], options); + } + // Normalize option `comment` + if(options.comment === undefined || options.comment === null || options.comment === false || options.comment === ''){ + options.comment = null; + }else { + if(typeof options.comment === 'string'){ + options.comment = Buffer.from(options.comment, options.encoding); + } + if(!isBuffer(options.comment)){ + throw new CsvError('CSV_INVALID_OPTION_COMMENT', [ + 'Invalid option comment:', + 'comment must be a buffer or a string,', + `got ${JSON.stringify(options.comment)}` ], options); } - // Normalize option `columns` - let fnFirstLineToHeaders = null; - if(options.columns === true){ - // Fields in the first line are converted as-is to columns - fnFirstLineToHeaders = undefined; - }else if(typeof options.columns === 'function'){ - fnFirstLineToHeaders = options.columns; - options.columns = true; - }else if(Array.isArray(options.columns)){ - options.columns = normalizeColumnsArray(options.columns); - }else if(options.columns === undefined || options.columns === null || options.columns === false){ - options.columns = false; - }else { - throw new CsvError('CSV_INVALID_OPTION_COLUMNS', [ - 'Invalid option columns:', - 'expect an array, a function or true,', - `got ${JSON.stringify(options.columns)}` - ], options); + } + // Normalize option `delimiter` + const delimiter_json = JSON.stringify(options.delimiter); + if(!Array.isArray(options.delimiter)) options.delimiter = [options.delimiter]; + if(options.delimiter.length === 0){ + throw new CsvError('CSV_INVALID_OPTION_DELIMITER', [ + 'Invalid option delimiter:', + 'delimiter must be a non empty string or buffer or array of string|buffer,', + `got ${delimiter_json}` + ], options); + } + options.delimiter = options.delimiter.map(function(delimiter){ + if(delimiter === undefined || delimiter === null || delimiter === false){ + return Buffer.from(',', options.encoding); } - // Normalize option `group_columns_by_name` - if(options.group_columns_by_name === undefined || options.group_columns_by_name === null || options.group_columns_by_name === false){ - options.group_columns_by_name = false; - }else if(options.group_columns_by_name !== true){ - throw new CsvError('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ - 'Invalid option group_columns_by_name:', - 'expect an boolean,', - `got ${JSON.stringify(options.group_columns_by_name)}` - ], options); - }else if(options.columns === false){ - throw new CsvError('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ - 'Invalid option group_columns_by_name:', - 'the `columns` mode must be activated.' - ], options); + if(typeof delimiter === 'string'){ + delimiter = Buffer.from(delimiter, options.encoding); } - // Normalize option `comment` - if(options.comment === undefined || options.comment === null || options.comment === false || options.comment === ''){ - options.comment = null; - }else { - if(typeof options.comment === 'string'){ - options.comment = Buffer.from(options.comment, options.encoding); - } - if(!isBuffer(options.comment)){ - throw new CsvError('CSV_INVALID_OPTION_COMMENT', [ - 'Invalid option comment:', - 'comment must be a buffer or a string,', - `got ${JSON.stringify(options.comment)}` - ], options); - } - } - // Normalize option `delimiter` - const delimiter_json = JSON.stringify(options.delimiter); - if(!Array.isArray(options.delimiter)) options.delimiter = [options.delimiter]; - if(options.delimiter.length === 0){ + if(!isBuffer(delimiter) || delimiter.length === 0){ throw new CsvError('CSV_INVALID_OPTION_DELIMITER', [ 'Invalid option delimiter:', 'delimiter must be a non empty string or buffer or array of string|buffer,', `got ${delimiter_json}` ], options); } - options.delimiter = options.delimiter.map(function(delimiter){ - if(delimiter === undefined || delimiter === null || delimiter === false){ - return Buffer.from(',', options.encoding); - } - if(typeof delimiter === 'string'){ - delimiter = Buffer.from(delimiter, options.encoding); - } - if(!isBuffer(delimiter) || delimiter.length === 0){ - throw new CsvError('CSV_INVALID_OPTION_DELIMITER', [ - 'Invalid option delimiter:', - 'delimiter must be a non empty string or buffer or array of string|buffer,', - `got ${delimiter_json}` - ], options); - } - return delimiter; - }); - // Normalize option `escape` - if(options.escape === undefined || options.escape === true){ - options.escape = Buffer.from('"', options.encoding); - }else if(typeof options.escape === 'string'){ - options.escape = Buffer.from(options.escape, options.encoding); - }else if (options.escape === null || options.escape === false){ - options.escape = null; - } - if(options.escape !== null){ - if(!isBuffer(options.escape)){ - throw new Error(`Invalid Option: escape must be a buffer, a string or a boolean, got ${JSON.stringify(options.escape)}`); + return delimiter; + }); + // Normalize option `escape` + if(options.escape === undefined || options.escape === true){ + options.escape = Buffer.from('"', options.encoding); + }else if(typeof options.escape === 'string'){ + options.escape = Buffer.from(options.escape, options.encoding); + }else if (options.escape === null || options.escape === false){ + options.escape = null; + } + if(options.escape !== null){ + if(!isBuffer(options.escape)){ + throw new Error(`Invalid Option: escape must be a buffer, a string or a boolean, got ${JSON.stringify(options.escape)}`); + } + } + // Normalize option `from` + if(options.from === undefined || options.from === null){ + options.from = 1; + }else { + if(typeof options.from === 'string' && /\d+/.test(options.from)){ + options.from = parseInt(options.from); + } + if(Number.isInteger(options.from)){ + if(options.from < 0){ + throw new Error(`Invalid Option: from must be a positive integer, got ${JSON.stringify(opts.from)}`); } - } - // Normalize option `from` - if(options.from === undefined || options.from === null){ - options.from = 1; }else { - if(typeof options.from === 'string' && /\d+/.test(options.from)){ - options.from = parseInt(options.from); - } - if(Number.isInteger(options.from)){ - if(options.from < 0){ - throw new Error(`Invalid Option: from must be a positive integer, got ${JSON.stringify(opts.from)}`); - } - }else { - throw new Error(`Invalid Option: from must be an integer, got ${JSON.stringify(options.from)}`); - } + throw new Error(`Invalid Option: from must be an integer, got ${JSON.stringify(options.from)}`); } - // Normalize option `from_line` - if(options.from_line === undefined || options.from_line === null){ - options.from_line = 1; - }else { - if(typeof options.from_line === 'string' && /\d+/.test(options.from_line)){ - options.from_line = parseInt(options.from_line); - } - if(Number.isInteger(options.from_line)){ - if(options.from_line <= 0){ - throw new Error(`Invalid Option: from_line must be a positive integer greater than 0, got ${JSON.stringify(opts.from_line)}`); - } - }else { - throw new Error(`Invalid Option: from_line must be an integer, got ${JSON.stringify(opts.from_line)}`); - } + } + // Normalize option `from_line` + if(options.from_line === undefined || options.from_line === null){ + options.from_line = 1; + }else { + if(typeof options.from_line === 'string' && /\d+/.test(options.from_line)){ + options.from_line = parseInt(options.from_line); } - // Normalize options `ignore_last_delimiters` - if(options.ignore_last_delimiters === undefined || options.ignore_last_delimiters === null){ - options.ignore_last_delimiters = false; - }else if(typeof options.ignore_last_delimiters === 'number'){ - options.ignore_last_delimiters = Math.floor(options.ignore_last_delimiters); - if(options.ignore_last_delimiters === 0){ - options.ignore_last_delimiters = false; + if(Number.isInteger(options.from_line)){ + if(options.from_line <= 0){ + throw new Error(`Invalid Option: from_line must be a positive integer greater than 0, got ${JSON.stringify(opts.from_line)}`); } - }else if(typeof options.ignore_last_delimiters !== 'boolean'){ - throw new CsvError('CSV_INVALID_OPTION_IGNORE_LAST_DELIMITERS', [ - 'Invalid option `ignore_last_delimiters`:', - 'the value must be a boolean value or an integer,', - `got ${JSON.stringify(options.ignore_last_delimiters)}` - ], options); - } - if(options.ignore_last_delimiters === true && options.columns === false){ - throw new CsvError('CSV_IGNORE_LAST_DELIMITERS_REQUIRES_COLUMNS', [ - 'The option `ignore_last_delimiters`', - 'requires the activation of the `columns` option' - ], options); - } - // Normalize option `info` - if(options.info === undefined || options.info === null || options.info === false){ - options.info = false; - }else if(options.info !== true){ - throw new Error(`Invalid Option: info must be true, got ${JSON.stringify(options.info)}`); - } - // Normalize option `max_record_size` - if(options.max_record_size === undefined || options.max_record_size === null || options.max_record_size === false){ - options.max_record_size = 0; - }else if(Number.isInteger(options.max_record_size) && options.max_record_size >= 0);else if(typeof options.max_record_size === 'string' && /\d+/.test(options.max_record_size)){ - options.max_record_size = parseInt(options.max_record_size); }else { - throw new Error(`Invalid Option: max_record_size must be a positive integer, got ${JSON.stringify(options.max_record_size)}`); - } - // Normalize option `objname` - if(options.objname === undefined || options.objname === null || options.objname === false){ - options.objname = undefined; - }else if(isBuffer(options.objname)){ - if(options.objname.length === 0){ - throw new Error(`Invalid Option: objname must be a non empty buffer`); - } - if(options.encoding === null);else { - options.objname = options.objname.toString(options.encoding); - } - }else if(typeof options.objname === 'string'){ - if(options.objname.length === 0){ - throw new Error(`Invalid Option: objname must be a non empty string`); - } - // Great, nothing to do - }else if(typeof options.objname === 'number');else { - throw new Error(`Invalid Option: objname must be a string or a buffer, got ${options.objname}`); - } - if(options.objname !== undefined){ - if(typeof options.objname === 'number'){ - if(options.columns !== false){ - throw Error('Invalid Option: objname index cannot be combined with columns or be defined as a field'); - } - }else { // A string or a buffer - if(options.columns === false){ - throw Error('Invalid Option: objname field must be combined with columns or be defined as an index'); - } - } + throw new Error(`Invalid Option: from_line must be an integer, got ${JSON.stringify(opts.from_line)}`); } - // Normalize option `on_record` - if(options.on_record === undefined || options.on_record === null){ - options.on_record = undefined; - }else if(typeof options.on_record !== 'function'){ - throw new CsvError('CSV_INVALID_OPTION_ON_RECORD', [ - 'Invalid option `on_record`:', - 'expect a function,', - `got ${JSON.stringify(options.on_record)}` - ], options); + } + // Normalize options `ignore_last_delimiters` + if(options.ignore_last_delimiters === undefined || options.ignore_last_delimiters === null){ + options.ignore_last_delimiters = false; + }else if(typeof options.ignore_last_delimiters === 'number'){ + options.ignore_last_delimiters = Math.floor(options.ignore_last_delimiters); + if(options.ignore_last_delimiters === 0){ + options.ignore_last_delimiters = false; } - // Normalize option `quote` - if(options.quote === null || options.quote === false || options.quote === ''){ - options.quote = null; - }else { - if(options.quote === undefined || options.quote === true){ - options.quote = Buffer.from('"', options.encoding); - }else if(typeof options.quote === 'string'){ - options.quote = Buffer.from(options.quote, options.encoding); + }else if(typeof options.ignore_last_delimiters !== 'boolean'){ + throw new CsvError('CSV_INVALID_OPTION_IGNORE_LAST_DELIMITERS', [ + 'Invalid option `ignore_last_delimiters`:', + 'the value must be a boolean value or an integer,', + `got ${JSON.stringify(options.ignore_last_delimiters)}` + ], options); + } + if(options.ignore_last_delimiters === true && options.columns === false){ + throw new CsvError('CSV_IGNORE_LAST_DELIMITERS_REQUIRES_COLUMNS', [ + 'The option `ignore_last_delimiters`', + 'requires the activation of the `columns` option' + ], options); + } + // Normalize option `info` + if(options.info === undefined || options.info === null || options.info === false){ + options.info = false; + }else if(options.info !== true){ + throw new Error(`Invalid Option: info must be true, got ${JSON.stringify(options.info)}`); + } + // Normalize option `max_record_size` + if(options.max_record_size === undefined || options.max_record_size === null || options.max_record_size === false){ + options.max_record_size = 0; + }else if(Number.isInteger(options.max_record_size) && options.max_record_size >= 0);else if(typeof options.max_record_size === 'string' && /\d+/.test(options.max_record_size)){ + options.max_record_size = parseInt(options.max_record_size); + }else { + throw new Error(`Invalid Option: max_record_size must be a positive integer, got ${JSON.stringify(options.max_record_size)}`); + } + // Normalize option `objname` + if(options.objname === undefined || options.objname === null || options.objname === false){ + options.objname = undefined; + }else if(isBuffer(options.objname)){ + if(options.objname.length === 0){ + throw new Error(`Invalid Option: objname must be a non empty buffer`); + } + if(options.encoding === null);else { + options.objname = options.objname.toString(options.encoding); + } + }else if(typeof options.objname === 'string'){ + if(options.objname.length === 0){ + throw new Error(`Invalid Option: objname must be a non empty string`); + } + // Great, nothing to do + }else if(typeof options.objname === 'number');else { + throw new Error(`Invalid Option: objname must be a string or a buffer, got ${options.objname}`); + } + if(options.objname !== undefined){ + if(typeof options.objname === 'number'){ + if(options.columns !== false){ + throw Error('Invalid Option: objname index cannot be combined with columns or be defined as a field'); } - if(!isBuffer(options.quote)){ - throw new Error(`Invalid Option: quote must be a buffer or a string, got ${JSON.stringify(options.quote)}`); + }else { // A string or a buffer + if(options.columns === false){ + throw Error('Invalid Option: objname field must be combined with columns or be defined as an index'); } } - // Normalize option `raw` - if(options.raw === undefined || options.raw === null || options.raw === false){ - options.raw = false; - }else if(options.raw !== true){ - throw new Error(`Invalid Option: raw must be true, got ${JSON.stringify(options.raw)}`); - } - // Normalize option `record_delimiter` - if(options.record_delimiter === undefined){ - options.record_delimiter = []; - }else if(typeof options.record_delimiter === 'string' || isBuffer(options.record_delimiter)){ - if(options.record_delimiter.length === 0){ - throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ - 'Invalid option `record_delimiter`:', - 'value must be a non empty string or buffer,', - `got ${JSON.stringify(options.record_delimiter)}` - ], options); - } - options.record_delimiter = [options.record_delimiter]; - }else if(!Array.isArray(options.record_delimiter)){ + } + // Normalize option `on_record` + if(options.on_record === undefined || options.on_record === null){ + options.on_record = undefined; + }else if(typeof options.on_record !== 'function'){ + throw new CsvError('CSV_INVALID_OPTION_ON_RECORD', [ + 'Invalid option `on_record`:', + 'expect a function,', + `got ${JSON.stringify(options.on_record)}` + ], options); + } + // Normalize option `quote` + if(options.quote === null || options.quote === false || options.quote === ''){ + options.quote = null; + }else { + if(options.quote === undefined || options.quote === true){ + options.quote = Buffer.from('"', options.encoding); + }else if(typeof options.quote === 'string'){ + options.quote = Buffer.from(options.quote, options.encoding); + } + if(!isBuffer(options.quote)){ + throw new Error(`Invalid Option: quote must be a buffer or a string, got ${JSON.stringify(options.quote)}`); + } + } + // Normalize option `raw` + if(options.raw === undefined || options.raw === null || options.raw === false){ + options.raw = false; + }else if(options.raw !== true){ + throw new Error(`Invalid Option: raw must be true, got ${JSON.stringify(options.raw)}`); + } + // Normalize option `record_delimiter` + if(options.record_delimiter === undefined){ + options.record_delimiter = []; + }else if(typeof options.record_delimiter === 'string' || isBuffer(options.record_delimiter)){ + if(options.record_delimiter.length === 0){ throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ 'Invalid option `record_delimiter`:', - 'value must be a string, a buffer or array of string|buffer,', + 'value must be a non empty string or buffer,', `got ${JSON.stringify(options.record_delimiter)}` ], options); } - options.record_delimiter = options.record_delimiter.map(function(rd, i){ - if(typeof rd !== 'string' && ! isBuffer(rd)){ - throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ - 'Invalid option `record_delimiter`:', - 'value must be a string, a buffer or array of string|buffer', - `at index ${i},`, - `got ${JSON.stringify(rd)}` - ], options); - }else if(rd.length === 0){ - throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ - 'Invalid option `record_delimiter`:', - 'value must be a non empty string or buffer', - `at index ${i},`, - `got ${JSON.stringify(rd)}` - ], options); - } - if(typeof rd === 'string'){ - rd = Buffer.from(rd, options.encoding); - } - return rd; - }); - // Normalize option `relax_column_count` - if(typeof options.relax_column_count === 'boolean');else if(options.relax_column_count === undefined || options.relax_column_count === null){ - options.relax_column_count = false; - }else { - throw new Error(`Invalid Option: relax_column_count must be a boolean, got ${JSON.stringify(options.relax_column_count)}`); - } - if(typeof options.relax_column_count_less === 'boolean');else if(options.relax_column_count_less === undefined || options.relax_column_count_less === null){ - options.relax_column_count_less = false; - }else { - throw new Error(`Invalid Option: relax_column_count_less must be a boolean, got ${JSON.stringify(options.relax_column_count_less)}`); + options.record_delimiter = [options.record_delimiter]; + }else if(!Array.isArray(options.record_delimiter)){ + throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ + 'Invalid option `record_delimiter`:', + 'value must be a string, a buffer or array of string|buffer,', + `got ${JSON.stringify(options.record_delimiter)}` + ], options); + } + options.record_delimiter = options.record_delimiter.map(function(rd, i){ + if(typeof rd !== 'string' && ! isBuffer(rd)){ + throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ + 'Invalid option `record_delimiter`:', + 'value must be a string, a buffer or array of string|buffer', + `at index ${i},`, + `got ${JSON.stringify(rd)}` + ], options); + }else if(rd.length === 0){ + throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ + 'Invalid option `record_delimiter`:', + 'value must be a non empty string or buffer', + `at index ${i},`, + `got ${JSON.stringify(rd)}` + ], options); } - if(typeof options.relax_column_count_more === 'boolean');else if(options.relax_column_count_more === undefined || options.relax_column_count_more === null){ - options.relax_column_count_more = false; - }else { - throw new Error(`Invalid Option: relax_column_count_more must be a boolean, got ${JSON.stringify(options.relax_column_count_more)}`); + if(typeof rd === 'string'){ + rd = Buffer.from(rd, options.encoding); } - // Normalize option `relax_quotes` - if(typeof options.relax_quotes === 'boolean');else if(options.relax_quotes === undefined || options.relax_quotes === null){ - options.relax_quotes = false; + return rd; + }); + // Normalize option `relax_column_count` + if(typeof options.relax_column_count === 'boolean');else if(options.relax_column_count === undefined || options.relax_column_count === null){ + options.relax_column_count = false; + }else { + throw new Error(`Invalid Option: relax_column_count must be a boolean, got ${JSON.stringify(options.relax_column_count)}`); + } + if(typeof options.relax_column_count_less === 'boolean');else if(options.relax_column_count_less === undefined || options.relax_column_count_less === null){ + options.relax_column_count_less = false; + }else { + throw new Error(`Invalid Option: relax_column_count_less must be a boolean, got ${JSON.stringify(options.relax_column_count_less)}`); + } + if(typeof options.relax_column_count_more === 'boolean');else if(options.relax_column_count_more === undefined || options.relax_column_count_more === null){ + options.relax_column_count_more = false; + }else { + throw new Error(`Invalid Option: relax_column_count_more must be a boolean, got ${JSON.stringify(options.relax_column_count_more)}`); + } + // Normalize option `relax_quotes` + if(typeof options.relax_quotes === 'boolean');else if(options.relax_quotes === undefined || options.relax_quotes === null){ + options.relax_quotes = false; + }else { + throw new Error(`Invalid Option: relax_quotes must be a boolean, got ${JSON.stringify(options.relax_quotes)}`); + } + // Normalize option `skip_empty_lines` + if(typeof options.skip_empty_lines === 'boolean');else if(options.skip_empty_lines === undefined || options.skip_empty_lines === null){ + options.skip_empty_lines = false; + }else { + throw new Error(`Invalid Option: skip_empty_lines must be a boolean, got ${JSON.stringify(options.skip_empty_lines)}`); + } + // Normalize option `skip_records_with_empty_values` + if(typeof options.skip_records_with_empty_values === 'boolean');else if(options.skip_records_with_empty_values === undefined || options.skip_records_with_empty_values === null){ + options.skip_records_with_empty_values = false; + }else { + throw new Error(`Invalid Option: skip_records_with_empty_values must be a boolean, got ${JSON.stringify(options.skip_records_with_empty_values)}`); + } + // Normalize option `skip_records_with_error` + if(typeof options.skip_records_with_error === 'boolean');else if(options.skip_records_with_error === undefined || options.skip_records_with_error === null){ + options.skip_records_with_error = false; + }else { + throw new Error(`Invalid Option: skip_records_with_error must be a boolean, got ${JSON.stringify(options.skip_records_with_error)}`); + } + // Normalize option `rtrim` + if(options.rtrim === undefined || options.rtrim === null || options.rtrim === false){ + options.rtrim = false; + }else if(options.rtrim !== true){ + throw new Error(`Invalid Option: rtrim must be a boolean, got ${JSON.stringify(options.rtrim)}`); + } + // Normalize option `ltrim` + if(options.ltrim === undefined || options.ltrim === null || options.ltrim === false){ + options.ltrim = false; + }else if(options.ltrim !== true){ + throw new Error(`Invalid Option: ltrim must be a boolean, got ${JSON.stringify(options.ltrim)}`); + } + // Normalize option `trim` + if(options.trim === undefined || options.trim === null || options.trim === false){ + options.trim = false; + }else if(options.trim !== true){ + throw new Error(`Invalid Option: trim must be a boolean, got ${JSON.stringify(options.trim)}`); + } + // Normalize options `trim`, `ltrim` and `rtrim` + if(options.trim === true && opts.ltrim !== false){ + options.ltrim = true; + }else if(options.ltrim !== true){ + options.ltrim = false; + } + if(options.trim === true && opts.rtrim !== false){ + options.rtrim = true; + }else if(options.rtrim !== true){ + options.rtrim = false; + } + // Normalize option `to` + if(options.to === undefined || options.to === null){ + options.to = -1; + }else { + if(typeof options.to === 'string' && /\d+/.test(options.to)){ + options.to = parseInt(options.to); + } + if(Number.isInteger(options.to)){ + if(options.to <= 0){ + throw new Error(`Invalid Option: to must be a positive integer greater than 0, got ${JSON.stringify(opts.to)}`); + } }else { - throw new Error(`Invalid Option: relax_quotes must be a boolean, got ${JSON.stringify(options.relax_quotes)}`); + throw new Error(`Invalid Option: to must be an integer, got ${JSON.stringify(opts.to)}`); } - // Normalize option `skip_empty_lines` - if(typeof options.skip_empty_lines === 'boolean');else if(options.skip_empty_lines === undefined || options.skip_empty_lines === null){ - options.skip_empty_lines = false; - }else { - throw new Error(`Invalid Option: skip_empty_lines must be a boolean, got ${JSON.stringify(options.skip_empty_lines)}`); + } + // Normalize option `to_line` + if(options.to_line === undefined || options.to_line === null){ + options.to_line = -1; + }else { + if(typeof options.to_line === 'string' && /\d+/.test(options.to_line)){ + options.to_line = parseInt(options.to_line); } - // Normalize option `skip_records_with_empty_values` - if(typeof options.skip_records_with_empty_values === 'boolean');else if(options.skip_records_with_empty_values === undefined || options.skip_records_with_empty_values === null){ - options.skip_records_with_empty_values = false; + if(Number.isInteger(options.to_line)){ + if(options.to_line <= 0){ + throw new Error(`Invalid Option: to_line must be a positive integer greater than 0, got ${JSON.stringify(opts.to_line)}`); + } }else { - throw new Error(`Invalid Option: skip_records_with_empty_values must be a boolean, got ${JSON.stringify(options.skip_records_with_empty_values)}`); + throw new Error(`Invalid Option: to_line must be an integer, got ${JSON.stringify(opts.to_line)}`); } - // Normalize option `skip_records_with_error` - if(typeof options.skip_records_with_error === 'boolean');else if(options.skip_records_with_error === undefined || options.skip_records_with_error === null){ - options.skip_records_with_error = false; - }else { - throw new Error(`Invalid Option: skip_records_with_error must be a boolean, got ${JSON.stringify(options.skip_records_with_error)}`); - } - // Normalize option `rtrim` - if(options.rtrim === undefined || options.rtrim === null || options.rtrim === false){ - options.rtrim = false; - }else if(options.rtrim !== true){ - throw new Error(`Invalid Option: rtrim must be a boolean, got ${JSON.stringify(options.rtrim)}`); - } - // Normalize option `ltrim` - if(options.ltrim === undefined || options.ltrim === null || options.ltrim === false){ - options.ltrim = false; - }else if(options.ltrim !== true){ - throw new Error(`Invalid Option: ltrim must be a boolean, got ${JSON.stringify(options.ltrim)}`); - } - // Normalize option `trim` - if(options.trim === undefined || options.trim === null || options.trim === false){ - options.trim = false; - }else if(options.trim !== true){ - throw new Error(`Invalid Option: trim must be a boolean, got ${JSON.stringify(options.trim)}`); - } - // Normalize options `trim`, `ltrim` and `rtrim` - if(options.trim === true && opts.ltrim !== false){ - options.ltrim = true; - }else if(options.ltrim !== true){ - options.ltrim = false; - } - if(options.trim === true && opts.rtrim !== false){ - options.rtrim = true; - }else if(options.rtrim !== true){ - options.rtrim = false; - } - // Normalize option `to` - if(options.to === undefined || options.to === null){ - options.to = -1; - }else { - if(typeof options.to === 'string' && /\d+/.test(options.to)){ - options.to = parseInt(options.to); - } - if(Number.isInteger(options.to)){ - if(options.to <= 0){ - throw new Error(`Invalid Option: to must be a positive integer greater than 0, got ${JSON.stringify(opts.to)}`); + } + return options; + }; + + const isRecordEmpty = function(record){ + return record.every((field) => field == null || field.toString && field.toString().trim() === ''); + }; + + // white space characters + // https://en.wikipedia.org/wiki/Whitespace_character + // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions/Character_Classes#Types + // \f\n\r\t\v\u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff + const tab = 9; + const nl = 10; // \n, 0x0A in hexadecimal, 10 in decimal + const np = 12; + const cr = 13; // \r, 0x0D in hexadécimal, 13 in decimal + const space = 32; + const boms = { + // Note, the following are equals: + // Buffer.from("\ufeff") + // Buffer.from([239, 187, 191]) + // Buffer.from('EFBBBF', 'hex') + 'utf8': Buffer.from([239, 187, 191]), + // Note, the following are equals: + // Buffer.from "\ufeff", 'utf16le + // Buffer.from([255, 254]) + 'utf16le': Buffer.from([255, 254]) + }; + + const transform = function(original_options = {}) { + const info = { + bytes: 0, + comment_lines: 0, + empty_lines: 0, + invalid_field_length: 0, + lines: 1, + records: 0 + }; + const options = normalize_options(original_options); + return { + info: info, + original_options: original_options, + options: options, + state: init_state(options), + __needMoreData: function(i, bufLen, end){ + if(end) return false; + const {quote} = this.options; + const {quoting, needMoreDataSize, recordDelimiterMaxLength} = this.state; + const numOfCharLeft = bufLen - i - 1; + const requiredLength = Math.max( + needMoreDataSize, + // Skip if the remaining buffer smaller than record delimiter + recordDelimiterMaxLength, + // Skip if the remaining buffer can be record delimiter following the closing quote + // 1 is for quote.length + quoting ? (quote.length + recordDelimiterMaxLength) : 0, + ); + return numOfCharLeft < requiredLength; + }, + // Central parser implementation + parse: function(nextBuf, end, push, close){ + const {bom, comment, escape, from_line, ltrim, max_record_size, quote, raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options; + let {record_delimiter} = this.options; + const {bomSkipped, previousBuf, rawBuffer, escapeIsQuote} = this.state; + let buf; + if(previousBuf === undefined){ + if(nextBuf === undefined){ + // Handle empty string + close(); + return; + }else { + buf = nextBuf; } + }else if(previousBuf !== undefined && nextBuf === undefined){ + buf = previousBuf; }else { - throw new Error(`Invalid Option: to must be an integer, got ${JSON.stringify(opts.to)}`); + buf = Buffer.concat([previousBuf, nextBuf]); } - } - // Normalize option `to_line` - if(options.to_line === undefined || options.to_line === null){ - options.to_line = -1; - }else { - if(typeof options.to_line === 'string' && /\d+/.test(options.to_line)){ - options.to_line = parseInt(options.to_line); - } - if(Number.isInteger(options.to_line)){ - if(options.to_line <= 0){ - throw new Error(`Invalid Option: to_line must be a positive integer greater than 0, got ${JSON.stringify(opts.to_line)}`); + // Handle UTF BOM + if(bomSkipped === false){ + if(bom === false){ + this.state.bomSkipped = true; + }else if(buf.length < 3){ + // No enough data + if(end === false){ + // Wait for more data + this.state.previousBuf = buf; + return; + } + }else { + for(const encoding in boms){ + if(boms[encoding].compare(buf, 0, boms[encoding].length) === 0){ + // Skip BOM + const bomLength = boms[encoding].length; + this.state.bufBytesStart += bomLength; + buf = buf.slice(bomLength); + // Renormalize original options with the new encoding + this.options = normalize_options({...this.original_options, encoding: encoding}); + break; + } + } + this.state.bomSkipped = true; } - }else { - throw new Error(`Invalid Option: to_line must be an integer, got ${JSON.stringify(opts.to_line)}`); } - } - this.info = { - bytes: 0, - comment_lines: 0, - empty_lines: 0, - invalid_field_length: 0, - lines: 1, - records: 0 - }; - this.options = options; - this.state = { - bomSkipped: false, - bufBytesStart: 0, - castField: fnCastField, - commenting: false, - // Current error encountered by a record - error: undefined, - enabled: options.from_line === 1, - escaping: false, - escapeIsQuote: isBuffer(options.escape) && isBuffer(options.quote) && Buffer.compare(options.escape, options.quote) === 0, - // columns can be `false`, `true`, `Array` - expectedRecordLength: Array.isArray(options.columns) ? options.columns.length : undefined, - field: new ResizeableBuffer(20), - firstLineToHeaders: fnFirstLineToHeaders, - needMoreDataSize: Math.max( - // Skip if the remaining buffer smaller than comment - options.comment !== null ? options.comment.length : 0, - // Skip if the remaining buffer can be delimiter - ...options.delimiter.map((delimiter) => delimiter.length), - // Skip if the remaining buffer can be escape sequence - options.quote !== null ? options.quote.length : 0, - ), - previousBuf: undefined, - quoting: false, - stop: false, - rawBuffer: new ResizeableBuffer(100), - record: [], - recordHasError: false, - record_length: 0, - recordDelimiterMaxLength: options.record_delimiter.length === 0 ? 2 : Math.max(...options.record_delimiter.map((v) => v.length)), - trimChars: [Buffer.from(' ', options.encoding)[0], Buffer.from('\t', options.encoding)[0]], - wasQuoting: false, - wasRowDelimiter: false - }; - } - // Implementation of `Transform._transform` - _transform(buf, encoding, callback){ - if(this.state.stop === true){ - return; - } - const err = this.__parse(buf, false); - if(err !== undefined){ - this.state.stop = true; - } - callback(err); - } - // Implementation of `Transform._flush` - _flush(callback){ - if(this.state.stop === true){ - return; - } - const err = this.__parse(undefined, true); - callback(err); - } - // Central parser implementation - __parse(nextBuf, end){ - const {bom, comment, escape, from_line, ltrim, max_record_size, quote, raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options; - let {record_delimiter} = this.options; - const {bomSkipped, previousBuf, rawBuffer, escapeIsQuote} = this.state; - let buf; - if(previousBuf === undefined){ - if(nextBuf === undefined){ - // Handle empty string - this.push(null); - return; - }else { - buf = nextBuf; - } - }else if(previousBuf !== undefined && nextBuf === undefined){ - buf = previousBuf; - }else { - buf = Buffer.concat([previousBuf, nextBuf]); - } - // Handle UTF BOM - if(bomSkipped === false){ - if(bom === false){ - this.state.bomSkipped = true; - }else if(buf.length < 3){ - // No enough data - if(end === false){ - // Wait for more data - this.state.previousBuf = buf; + const bufLen = buf.length; + let pos; + for(pos = 0; pos < bufLen; pos++){ + // Ensure we get enough space to look ahead + // There should be a way to move this out of the loop + if(this.__needMoreData(pos, bufLen, end)){ + break; + } + if(this.state.wasRowDelimiter === true){ + this.info.lines++; + this.state.wasRowDelimiter = false; + } + if(to_line !== -1 && this.info.lines > to_line){ + this.state.stop = true; + close(); return; } - }else { - for(const encoding in boms){ - if(boms[encoding].compare(buf, 0, boms[encoding].length) === 0){ - // Skip BOM - const bomLength = boms[encoding].length; - this.state.bufBytesStart += bomLength; - buf = buf.slice(bomLength); - // Renormalize original options with the new encoding - this.__normalizeOptions({...this.__originalOptions, encoding: encoding}); - break; + // Auto discovery of record_delimiter, unix, mac and windows supported + if(this.state.quoting === false && record_delimiter.length === 0){ + const record_delimiterCount = this.__autoDiscoverRecordDelimiter(buf, pos); + if(record_delimiterCount){ + record_delimiter = this.options.record_delimiter; } } - this.state.bomSkipped = true; - } - } - const bufLen = buf.length; - let pos; - for(pos = 0; pos < bufLen; pos++){ - // Ensure we get enough space to look ahead - // There should be a way to move this out of the loop - if(this.__needMoreData(pos, bufLen, end)){ - break; - } - if(this.state.wasRowDelimiter === true){ - this.info.lines++; - this.state.wasRowDelimiter = false; - } - if(to_line !== -1 && this.info.lines > to_line){ - this.state.stop = true; - this.push(null); - return; - } - // Auto discovery of record_delimiter, unix, mac and windows supported - if(this.state.quoting === false && record_delimiter.length === 0){ - const record_delimiterCount = this.__autoDiscoverRecordDelimiter(buf, pos); - if(record_delimiterCount){ - record_delimiter = this.options.record_delimiter; + const chr = buf[pos]; + if(raw === true){ + rawBuffer.append(chr); } - } - const chr = buf[pos]; - if(raw === true){ - rawBuffer.append(chr); - } - if((chr === cr || chr === nl) && this.state.wasRowDelimiter === false){ - this.state.wasRowDelimiter = true; - } - // Previous char was a valid escape char - // treat the current char as a regular char - if(this.state.escaping === true){ - this.state.escaping = false; - }else { - // Escape is only active inside quoted fields - // We are quoting, the char is an escape chr and there is a chr to escape - // if(escape !== null && this.state.quoting === true && chr === escape && pos + 1 < bufLen){ - if(escape !== null && this.state.quoting === true && this.__isEscape(buf, pos, chr) && pos + escape.length < bufLen){ - if(escapeIsQuote){ - if(this.__isQuote(buf, pos+escape.length)){ + if((chr === cr || chr === nl) && this.state.wasRowDelimiter === false){ + this.state.wasRowDelimiter = true; + } + // Previous char was a valid escape char + // treat the current char as a regular char + if(this.state.escaping === true){ + this.state.escaping = false; + }else { + // Escape is only active inside quoted fields + // We are quoting, the char is an escape chr and there is a chr to escape + // if(escape !== null && this.state.quoting === true && chr === escape && pos + 1 < bufLen){ + if(escape !== null && this.state.quoting === true && this.__isEscape(buf, pos, chr) && pos + escape.length < bufLen){ + if(escapeIsQuote){ + if(this.__isQuote(buf, pos+escape.length)){ + this.state.escaping = true; + pos += escape.length - 1; + continue; + } + }else { this.state.escaping = true; pos += escape.length - 1; continue; } - }else { - this.state.escaping = true; - pos += escape.length - 1; - continue; } - } - // Not currently escaping and chr is a quote - // TODO: need to compare bytes instead of single char - if(this.state.commenting === false && this.__isQuote(buf, pos)){ - if(this.state.quoting === true){ - const nextChr = buf[pos+quote.length]; - const isNextChrTrimable = rtrim && this.__isCharTrimable(nextChr); - const isNextChrComment = comment !== null && this.__compareBytes(comment, buf, pos+quote.length, nextChr); - const isNextChrDelimiter = this.__isDelimiter(buf, pos+quote.length, nextChr); - const isNextChrRecordDelimiter = record_delimiter.length === 0 ? this.__autoDiscoverRecordDelimiter(buf, pos+quote.length) : this.__isRecordDelimiter(nextChr, buf, pos+quote.length); - // Escape a quote - // Treat next char as a regular character - if(escape !== null && this.__isEscape(buf, pos, chr) && this.__isQuote(buf, pos + escape.length)){ - pos += escape.length - 1; - }else if(!nextChr || isNextChrDelimiter || isNextChrRecordDelimiter || isNextChrComment || isNextChrTrimable){ - this.state.quoting = false; - this.state.wasQuoting = true; - pos += quote.length - 1; - continue; - }else if(relax_quotes === false){ - const err = this.__error( - new CsvError('CSV_INVALID_CLOSING_QUOTE', [ - 'Invalid Closing Quote:', - `got "${String.fromCharCode(nextChr)}"`, - `at line ${this.info.lines}`, - 'instead of delimiter, record delimiter, trimable character', - '(if activated) or comment', - ], this.options, this.__infoField()) - ); - if(err !== undefined) return err; - }else { - this.state.quoting = false; - this.state.wasQuoting = true; - this.state.field.prepend(quote); - pos += quote.length - 1; - } - }else { - if(this.state.field.length !== 0){ - // In relax_quotes mode, treat opening quote preceded by chrs as regular - if(relax_quotes === false){ + // Not currently escaping and chr is a quote + // TODO: need to compare bytes instead of single char + if(this.state.commenting === false && this.__isQuote(buf, pos)){ + if(this.state.quoting === true){ + const nextChr = buf[pos+quote.length]; + const isNextChrTrimable = rtrim && this.__isCharTrimable(nextChr); + const isNextChrComment = comment !== null && this.__compareBytes(comment, buf, pos+quote.length, nextChr); + const isNextChrDelimiter = this.__isDelimiter(buf, pos+quote.length, nextChr); + const isNextChrRecordDelimiter = record_delimiter.length === 0 ? this.__autoDiscoverRecordDelimiter(buf, pos+quote.length) : this.__isRecordDelimiter(nextChr, buf, pos+quote.length); + // Escape a quote + // Treat next char as a regular character + if(escape !== null && this.__isEscape(buf, pos, chr) && this.__isQuote(buf, pos + escape.length)){ + pos += escape.length - 1; + }else if(!nextChr || isNextChrDelimiter || isNextChrRecordDelimiter || isNextChrComment || isNextChrTrimable){ + this.state.quoting = false; + this.state.wasQuoting = true; + pos += quote.length - 1; + continue; + }else if(relax_quotes === false){ const err = this.__error( - new CsvError('INVALID_OPENING_QUOTE', [ - 'Invalid Opening Quote:', - `a quote is found inside a field at line ${this.info.lines}`, - ], this.options, this.__infoField(), { - field: this.state.field, - }) + new CsvError('CSV_INVALID_CLOSING_QUOTE', [ + 'Invalid Closing Quote:', + `got "${String.fromCharCode(nextChr)}"`, + `at line ${this.info.lines}`, + 'instead of delimiter, record delimiter, trimable character', + '(if activated) or comment', + ], this.options, this.__infoField()) ); if(err !== undefined) return err; + }else { + this.state.quoting = false; + this.state.wasQuoting = true; + this.state.field.prepend(quote); + pos += quote.length - 1; } }else { - this.state.quoting = true; - pos += quote.length - 1; - continue; - } - } - } - if(this.state.quoting === false){ - const recordDelimiterLength = this.__isRecordDelimiter(chr, buf, pos); - if(recordDelimiterLength !== 0){ - // Do not emit comments which take a full line - const skipCommentLine = this.state.commenting && (this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0); - if(skipCommentLine){ - this.info.comment_lines++; - // Skip full comment line - }else { - // Activate records emition if above from_line - if(this.state.enabled === false && this.info.lines + (this.state.wasRowDelimiter === true ? 1: 0) >= from_line){ - this.state.enabled = true; - this.__resetField(); - this.__resetRecord(); - pos += recordDelimiterLength - 1; + if(this.state.field.length !== 0){ + // In relax_quotes mode, treat opening quote preceded by chrs as regular + if(relax_quotes === false){ + const err = this.__error( + new CsvError('INVALID_OPENING_QUOTE', [ + 'Invalid Opening Quote:', + `a quote is found inside a field at line ${this.info.lines}`, + ], this.options, this.__infoField(), { + field: this.state.field, + }) + ); + if(err !== undefined) return err; + } + }else { + this.state.quoting = true; + pos += quote.length - 1; continue; } - // Skip if line is empty and skip_empty_lines activated - if(skip_empty_lines === true && this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0){ - this.info.empty_lines++; - pos += recordDelimiterLength - 1; - continue; + } + } + if(this.state.quoting === false){ + const recordDelimiterLength = this.__isRecordDelimiter(chr, buf, pos); + if(recordDelimiterLength !== 0){ + // Do not emit comments which take a full line + const skipCommentLine = this.state.commenting && (this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0); + if(skipCommentLine){ + this.info.comment_lines++; + // Skip full comment line + }else { + // Activate records emition if above from_line + if(this.state.enabled === false && this.info.lines + (this.state.wasRowDelimiter === true ? 1: 0) >= from_line){ + this.state.enabled = true; + this.__resetField(); + this.__resetRecord(); + pos += recordDelimiterLength - 1; + continue; + } + // Skip if line is empty and skip_empty_lines activated + if(skip_empty_lines === true && this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0){ + this.info.empty_lines++; + pos += recordDelimiterLength - 1; + continue; + } + this.info.bytes = this.state.bufBytesStart + pos; + const errField = this.__onField(); + if(errField !== undefined) return errField; + this.info.bytes = this.state.bufBytesStart + pos + recordDelimiterLength; + const errRecord = this.__onRecord(push); + if(errRecord !== undefined) return errRecord; + if(to !== -1 && this.info.records >= to){ + this.state.stop = true; + close(); + return; + } } + this.state.commenting = false; + pos += recordDelimiterLength - 1; + continue; + } + if(this.state.commenting){ + continue; + } + const commentCount = comment === null ? 0 : this.__compareBytes(comment, buf, pos, chr); + if(commentCount !== 0){ + this.state.commenting = true; + continue; + } + const delimiterLength = this.__isDelimiter(buf, pos, chr); + if(delimiterLength !== 0){ this.info.bytes = this.state.bufBytesStart + pos; const errField = this.__onField(); if(errField !== undefined) return errField; - this.info.bytes = this.state.bufBytesStart + pos + recordDelimiterLength; - const errRecord = this.__onRecord(); - if(errRecord !== undefined) return errRecord; - if(to !== -1 && this.info.records >= to){ - this.state.stop = true; - this.push(null); - return; - } + pos += delimiterLength - 1; + continue; } - this.state.commenting = false; - pos += recordDelimiterLength - 1; - continue; - } - if(this.state.commenting){ - continue; } - const commentCount = comment === null ? 0 : this.__compareBytes(comment, buf, pos, chr); - if(commentCount !== 0){ - this.state.commenting = true; - continue; - } - const delimiterLength = this.__isDelimiter(buf, pos, chr); - if(delimiterLength !== 0){ - this.info.bytes = this.state.bufBytesStart + pos; - const errField = this.__onField(); - if(errField !== undefined) return errField; - pos += delimiterLength - 1; - continue; + } + if(this.state.commenting === false){ + if(max_record_size !== 0 && this.state.record_length + this.state.field.length > max_record_size){ + const err = this.__error( + new CsvError('CSV_MAX_RECORD_SIZE', [ + 'Max Record Size:', + 'record exceed the maximum number of tolerated bytes', + `of ${max_record_size}`, + `at line ${this.info.lines}`, + ], this.options, this.__infoField()) + ); + if(err !== undefined) return err; } } - } - if(this.state.commenting === false){ - if(max_record_size !== 0 && this.state.record_length + this.state.field.length > max_record_size){ + const lappend = ltrim === false || this.state.quoting === true || this.state.field.length !== 0 || !this.__isCharTrimable(chr); + // rtrim in non quoting is handle in __onField + const rappend = rtrim === false || this.state.wasQuoting === false; + if(lappend === true && rappend === true){ + this.state.field.append(chr); + }else if(rtrim === true && !this.__isCharTrimable(chr)){ const err = this.__error( - new CsvError('CSV_MAX_RECORD_SIZE', [ - 'Max Record Size:', - 'record exceed the maximum number of tolerated bytes', - `of ${max_record_size}`, + new CsvError('CSV_NON_TRIMABLE_CHAR_AFTER_CLOSING_QUOTE', [ + 'Invalid Closing Quote:', + 'found non trimable byte after quote', `at line ${this.info.lines}`, ], this.options, this.__infoField()) ); if(err !== undefined) return err; } } - const lappend = ltrim === false || this.state.quoting === true || this.state.field.length !== 0 || !this.__isCharTrimable(chr); - // rtrim in non quoting is handle in __onField - const rappend = rtrim === false || this.state.wasQuoting === false; - if(lappend === true && rappend === true){ - this.state.field.append(chr); - }else if(rtrim === true && !this.__isCharTrimable(chr)){ - const err = this.__error( - new CsvError('CSV_NON_TRIMABLE_CHAR_AFTER_CLOSING_QUOTE', [ - 'Invalid Closing Quote:', - 'found non trimable byte after quote', - `at line ${this.info.lines}`, - ], this.options, this.__infoField()) - ); - if(err !== undefined) return err; - } - } - if(end === true){ - // Ensure we are not ending in a quoting state - if(this.state.quoting === true){ - const err = this.__error( - new CsvError('CSV_QUOTE_NOT_CLOSED', [ - 'Quote Not Closed:', - `the parsing is finished with an opening quote at line ${this.info.lines}`, - ], this.options, this.__infoField()) - ); - if(err !== undefined) return err; + if(end === true){ + // Ensure we are not ending in a quoting state + if(this.state.quoting === true){ + const err = this.__error( + new CsvError('CSV_QUOTE_NOT_CLOSED', [ + 'Quote Not Closed:', + `the parsing is finished with an opening quote at line ${this.info.lines}`, + ], this.options, this.__infoField()) + ); + if(err !== undefined) return err; + }else { + // Skip last line if it has no characters + if(this.state.wasQuoting === true || this.state.record.length !== 0 || this.state.field.length !== 0){ + this.info.bytes = this.state.bufBytesStart + pos; + const errField = this.__onField(); + if(errField !== undefined) return errField; + const errRecord = this.__onRecord(push); + if(errRecord !== undefined) return errRecord; + }else if(this.state.wasRowDelimiter === true){ + this.info.empty_lines++; + }else if(this.state.commenting === true){ + this.info.comment_lines++; + } + } }else { - // Skip last line if it has no characters - if(this.state.wasQuoting === true || this.state.record.length !== 0 || this.state.field.length !== 0){ - this.info.bytes = this.state.bufBytesStart + pos; - const errField = this.__onField(); - if(errField !== undefined) return errField; - const errRecord = this.__onRecord(); - if(errRecord !== undefined) return errRecord; - }else if(this.state.wasRowDelimiter === true){ - this.info.empty_lines++; - }else if(this.state.commenting === true){ - this.info.comment_lines++; + this.state.bufBytesStart += pos; + this.state.previousBuf = buf.slice(pos); + } + if(this.state.wasRowDelimiter === true){ + this.info.lines++; + this.state.wasRowDelimiter = false; + } + }, + __onRecord: function(push){ + const {columns, group_columns_by_name, encoding, info, from, relax_column_count, relax_column_count_less, relax_column_count_more, raw, skip_records_with_empty_values} = this.options; + const {enabled, record} = this.state; + if(enabled === false){ + return this.__resetRecord(); + } + // Convert the first line into column names + const recordLength = record.length; + if(columns === true){ + if(skip_records_with_empty_values === true && isRecordEmpty(record)){ + this.__resetRecord(); + return; + } + return this.__firstLineToColumns(record); + } + if(columns === false && this.info.records === 0){ + this.state.expectedRecordLength = recordLength; + } + if(recordLength !== this.state.expectedRecordLength){ + const err = columns === false ? + new CsvError('CSV_RECORD_INCONSISTENT_FIELDS_LENGTH', [ + 'Invalid Record Length:', + `expect ${this.state.expectedRecordLength},`, + `got ${recordLength} on line ${this.info.lines}`, + ], this.options, this.__infoField(), { + record: record, + }) + : + new CsvError('CSV_RECORD_INCONSISTENT_COLUMNS', [ + 'Invalid Record Length:', + `columns length is ${columns.length},`, // rename columns + `got ${recordLength} on line ${this.info.lines}`, + ], this.options, this.__infoField(), { + record: record, + }); + if(relax_column_count === true || + (relax_column_count_less === true && recordLength < this.state.expectedRecordLength) || + (relax_column_count_more === true && recordLength > this.state.expectedRecordLength)){ + this.info.invalid_field_length++; + this.state.error = err; + // Error is undefined with skip_records_with_error + }else { + const finalErr = this.__error(err); + if(finalErr) return finalErr; } } - }else { - this.state.bufBytesStart += pos; - this.state.previousBuf = buf.slice(pos); - } - if(this.state.wasRowDelimiter === true){ - this.info.lines++; - this.state.wasRowDelimiter = false; - } - } - __onRecord(){ - const {columns, group_columns_by_name, encoding, info, from, relax_column_count, relax_column_count_less, relax_column_count_more, raw, skip_records_with_empty_values} = this.options; - const {enabled, record} = this.state; - if(enabled === false){ - return this.__resetRecord(); - } - // Convert the first line into column names - const recordLength = record.length; - if(columns === true){ if(skip_records_with_empty_values === true && isRecordEmpty(record)){ this.__resetRecord(); return; } - return this.__firstLineToColumns(record); - } - if(columns === false && this.info.records === 0){ - this.state.expectedRecordLength = recordLength; - } - if(recordLength !== this.state.expectedRecordLength){ - const err = columns === false ? - new CsvError('CSV_RECORD_INCONSISTENT_FIELDS_LENGTH', [ - 'Invalid Record Length:', - `expect ${this.state.expectedRecordLength},`, - `got ${recordLength} on line ${this.info.lines}`, - ], this.options, this.__infoField(), { - record: record, - }) - : - new CsvError('CSV_RECORD_INCONSISTENT_COLUMNS', [ - 'Invalid Record Length:', - `columns length is ${columns.length},`, // rename columns - `got ${recordLength} on line ${this.info.lines}`, - ], this.options, this.__infoField(), { - record: record, - }); - if(relax_column_count === true || - (relax_column_count_less === true && recordLength < this.state.expectedRecordLength) || - (relax_column_count_more === true && recordLength > this.state.expectedRecordLength)){ - this.info.invalid_field_length++; - this.state.error = err; - // Error is undefined with skip_records_with_error - }else { - const finalErr = this.__error(err); - if(finalErr) return finalErr; + if(this.state.recordHasError === true){ + this.__resetRecord(); + this.state.recordHasError = false; + return; } - } - if(skip_records_with_empty_values === true && isRecordEmpty(record)){ - this.__resetRecord(); - return; - } - if(this.state.recordHasError === true){ - this.__resetRecord(); - this.state.recordHasError = false; - return; - } - this.info.records++; - if(from === 1 || this.info.records >= from){ - const {objname} = this.options; - // With columns, records are object - if(columns !== false){ - const obj = {}; - // Transform record array to an object - for(let i = 0, l = record.length; i < l; i++){ - if(columns[i] === undefined || columns[i].disabled) continue; - // Turn duplicate columns into an array - if (group_columns_by_name === true && obj[columns[i].name] !== undefined) { - if (Array.isArray(obj[columns[i].name])) { - obj[columns[i].name] = obj[columns[i].name].concat(record[i]); + this.info.records++; + if(from === 1 || this.info.records >= from){ + const {objname} = this.options; + // With columns, records are object + if(columns !== false){ + const obj = {}; + // Transform record array to an object + for(let i = 0, l = record.length; i < l; i++){ + if(columns[i] === undefined || columns[i].disabled) continue; + // Turn duplicate columns into an array + if (group_columns_by_name === true && obj[columns[i].name] !== undefined) { + if (Array.isArray(obj[columns[i].name])) { + obj[columns[i].name] = obj[columns[i].name].concat(record[i]); + } else { + obj[columns[i].name] = [obj[columns[i].name], record[i]]; + } } else { - obj[columns[i].name] = [obj[columns[i].name], record[i]]; + obj[columns[i].name] = record[i]; } - } else { - obj[columns[i].name] = record[i]; } - } - // Without objname (default) - if(raw === true || info === true){ - const extRecord = Object.assign( - {record: obj}, - (raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}), - (info === true ? {info: this.__infoRecord()}: {}) - ); - const err = this.__push( - objname === undefined ? extRecord : [obj[objname], extRecord] - ); - if(err){ - return err; - } - }else { - const err = this.__push( - objname === undefined ? obj : [obj[objname], obj] - ); - if(err){ - return err; - } - } - // Without columns, records are array - }else { - if(raw === true || info === true){ - const extRecord = Object.assign( - {record: record}, - raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}, - info === true ? {info: this.__infoRecord()}: {} - ); - const err = this.__push( - objname === undefined ? extRecord : [record[objname], extRecord] - ); - if(err){ - return err; + // Without objname (default) + if(raw === true || info === true){ + const extRecord = Object.assign( + {record: obj}, + (raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}), + (info === true ? {info: this.__infoRecord()}: {}) + ); + const err = this.__push( + objname === undefined ? extRecord : [obj[objname], extRecord] + , push); + if(err){ + return err; + } + }else { + const err = this.__push( + objname === undefined ? obj : [obj[objname], obj] + , push); + if(err){ + return err; + } } + // Without columns, records are array }else { - const err = this.__push( - objname === undefined ? record : [record[objname], record] - ); - if(err){ - return err; + if(raw === true || info === true){ + const extRecord = Object.assign( + {record: record}, + raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}, + info === true ? {info: this.__infoRecord()}: {} + ); + const err = this.__push( + objname === undefined ? extRecord : [record[objname], extRecord] + , push); + if(err){ + return err; + } + }else { + const err = this.__push( + objname === undefined ? record : [record[objname], record] + , push); + if(err){ + return err; + } } } } - } - this.__resetRecord(); - } - __firstLineToColumns(record){ - const {firstLineToHeaders} = this.state; - try{ - const headers = firstLineToHeaders === undefined ? record : firstLineToHeaders.call(null, record); - if(!Array.isArray(headers)){ - return this.__error( - new CsvError('CSV_INVALID_COLUMN_MAPPING', [ - 'Invalid Column Mapping:', - 'expect an array from column function,', - `got ${JSON.stringify(headers)}` - ], this.options, this.__infoField(), { - headers: headers, - }) - ); - } - const normalizedHeaders = normalizeColumnsArray(headers); - this.state.expectedRecordLength = normalizedHeaders.length; - this.options.columns = normalizedHeaders; this.__resetRecord(); - return; - }catch(err){ - return err; - } - } - __resetRecord(){ - if(this.options.raw === true){ - this.state.rawBuffer.reset(); - } - this.state.error = undefined; - this.state.record = []; - this.state.record_length = 0; - } - __onField(){ - const {cast, encoding, rtrim, max_record_size} = this.options; - const {enabled, wasQuoting} = this.state; - // Short circuit for the from_line options - if(enabled === false){ - return this.__resetField(); - } - let field = this.state.field.toString(encoding); - if(rtrim === true && wasQuoting === false){ - field = field.trimRight(); - } - if(cast === true){ - const [err, f] = this.__cast(field); - if(err !== undefined) return err; - field = f; - } - this.state.record.push(field); - // Increment record length if record size must not exceed a limit - if(max_record_size !== 0 && typeof field === 'string'){ - this.state.record_length += field.length; - } - this.__resetField(); - } - __resetField(){ - this.state.field.reset(); - this.state.wasQuoting = false; - } - __push(record){ - const {on_record} = this.options; - if(on_record !== undefined){ - const info = this.__infoRecord(); + }, + __firstLineToColumns: function(record){ + const {firstLineToHeaders} = this.state; try{ - record = on_record.call(null, record, info); + const headers = firstLineToHeaders === undefined ? record : firstLineToHeaders.call(null, record); + if(!Array.isArray(headers)){ + return this.__error( + new CsvError('CSV_INVALID_COLUMN_MAPPING', [ + 'Invalid Column Mapping:', + 'expect an array from column function,', + `got ${JSON.stringify(headers)}` + ], this.options, this.__infoField(), { + headers: headers, + }) + ); + } + const normalizedHeaders = normalize_columns_array(headers); + this.state.expectedRecordLength = normalizedHeaders.length; + this.options.columns = normalizedHeaders; + this.__resetRecord(); + return; }catch(err){ return err; } - if(record === undefined || record === null){ return; } - } - this.push(record); - } - // Return a tuple with the error and the casted value - __cast(field){ - const {columns, relax_column_count} = this.options; - const isColumns = Array.isArray(columns); - // Dont loose time calling cast - // because the final record is an object - // and this field can't be associated to a key present in columns - if(isColumns === true && relax_column_count && this.options.columns.length <= this.state.record.length){ - return [undefined, undefined]; - } - if(this.state.castField !== null){ - try{ + }, + __resetRecord: function(){ + if(this.options.raw === true){ + this.state.rawBuffer.reset(); + } + this.state.error = undefined; + this.state.record = []; + this.state.record_length = 0; + }, + __onField: function(){ + const {cast, encoding, rtrim, max_record_size} = this.options; + const {enabled, wasQuoting} = this.state; + // Short circuit for the from_line options + if(enabled === false){ + return this.__resetField(); + } + let field = this.state.field.toString(encoding); + if(rtrim === true && wasQuoting === false){ + field = field.trimRight(); + } + if(cast === true){ + const [err, f] = this.__cast(field); + if(err !== undefined) return err; + field = f; + } + this.state.record.push(field); + // Increment record length if record size must not exceed a limit + if(max_record_size !== 0 && typeof field === 'string'){ + this.state.record_length += field.length; + } + this.__resetField(); + }, + __resetField: function(){ + this.state.field.reset(); + this.state.wasQuoting = false; + }, + __push: function(record, push){ + const {on_record} = this.options; + if(on_record !== undefined){ + const info = this.__infoRecord(); + try{ + record = on_record.call(null, record, info); + }catch(err){ + return err; + } + if(record === undefined || record === null){ return; } + } + push(record); + }, + // Return a tuple with the error and the casted value + __cast: function(field){ + const {columns, relax_column_count} = this.options; + const isColumns = Array.isArray(columns); + // Dont loose time calling cast + // because the final record is an object + // and this field can't be associated to a key present in columns + if(isColumns === true && relax_column_count && this.options.columns.length <= this.state.record.length){ + return [undefined, undefined]; + } + if(this.state.castField !== null){ + try{ + const info = this.__infoField(); + return [undefined, this.state.castField.call(null, field, info)]; + }catch(err){ + return [err]; + } + } + if(this.__isFloat(field)){ + return [undefined, parseFloat(field)]; + }else if(this.options.cast_date !== false){ const info = this.__infoField(); - return [undefined, this.state.castField.call(null, field, info)]; - }catch(err){ - return [err]; + return [undefined, this.options.cast_date.call(null, field, info)]; } - } - if(this.__isFloat(field)){ - return [undefined, parseFloat(field)]; - }else if(this.options.cast_date !== false){ - const info = this.__infoField(); - return [undefined, this.options.cast_date.call(null, field, info)]; - } - return [undefined, field]; - } - // Helper to test if a character is a space or a line delimiter - __isCharTrimable(chr){ - return chr === space || chr === tab || chr === cr || chr === nl || chr === np; - } - // Keep it in case we implement the `cast_int` option - // __isInt(value){ - // // return Number.isInteger(parseInt(value)) - // // return !isNaN( parseInt( obj ) ); - // return /^(\-|\+)?[1-9][0-9]*$/.test(value) - // } - __isFloat(value){ - return (value - parseFloat(value) + 1) >= 0; // Borrowed from jquery - } - __compareBytes(sourceBuf, targetBuf, targetPos, firstByte){ - if(sourceBuf[0] !== firstByte) return 0; - const sourceLength = sourceBuf.length; - for(let i = 1; i < sourceLength; i++){ - if(sourceBuf[i] !== targetBuf[targetPos+i]) return 0; - } - return sourceLength; - } - __needMoreData(i, bufLen, end){ - if(end) return false; - const {quote} = this.options; - const {quoting, needMoreDataSize, recordDelimiterMaxLength} = this.state; - const numOfCharLeft = bufLen - i - 1; - const requiredLength = Math.max( - needMoreDataSize, - // Skip if the remaining buffer smaller than record delimiter - recordDelimiterMaxLength, - // Skip if the remaining buffer can be record delimiter following the closing quote - // 1 is for quote.length - quoting ? (quote.length + recordDelimiterMaxLength) : 0, - ); - return numOfCharLeft < requiredLength; - } - __isDelimiter(buf, pos, chr){ - const {delimiter, ignore_last_delimiters} = this.options; - if(ignore_last_delimiters === true && this.state.record.length === this.options.columns.length - 1){ - return 0; - }else if(ignore_last_delimiters !== false && typeof ignore_last_delimiters === 'number' && this.state.record.length === ignore_last_delimiters - 1){ - return 0; - } - loop1: for(let i = 0; i < delimiter.length; i++){ - const del = delimiter[i]; - if(del[0] === chr){ - for(let j = 1; j < del.length; j++){ - if(del[j] !== buf[pos+j]) continue loop1; + return [undefined, field]; + }, + // Helper to test if a character is a space or a line delimiter + __isCharTrimable: function(chr){ + return chr === space || chr === tab || chr === cr || chr === nl || chr === np; + }, + // Keep it in case we implement the `cast_int` option + // __isInt(value){ + // // return Number.isInteger(parseInt(value)) + // // return !isNaN( parseInt( obj ) ); + // return /^(\-|\+)?[1-9][0-9]*$/.test(value) + // } + __isFloat: function(value){ + return (value - parseFloat(value) + 1) >= 0; // Borrowed from jquery + }, + __compareBytes: function(sourceBuf, targetBuf, targetPos, firstByte){ + if(sourceBuf[0] !== firstByte) return 0; + const sourceLength = sourceBuf.length; + for(let i = 1; i < sourceLength; i++){ + if(sourceBuf[i] !== targetBuf[targetPos+i]) return 0; + } + return sourceLength; + }, + __isDelimiter: function(buf, pos, chr){ + const {delimiter, ignore_last_delimiters} = this.options; + if(ignore_last_delimiters === true && this.state.record.length === this.options.columns.length - 1){ + return 0; + }else if(ignore_last_delimiters !== false && typeof ignore_last_delimiters === 'number' && this.state.record.length === ignore_last_delimiters - 1){ + return 0; + } + loop1: for(let i = 0; i < delimiter.length; i++){ + const del = delimiter[i]; + if(del[0] === chr){ + for(let j = 1; j < del.length; j++){ + if(del[j] !== buf[pos+j]) continue loop1; + } + return del.length; } - return del.length; } - } - return 0; - } - __isRecordDelimiter(chr, buf, pos){ - const {record_delimiter} = this.options; - const recordDelimiterLength = record_delimiter.length; - loop1: for(let i = 0; i < recordDelimiterLength; i++){ - const rd = record_delimiter[i]; - const rdLength = rd.length; - if(rd[0] !== chr){ - continue; + return 0; + }, + __isRecordDelimiter: function(chr, buf, pos){ + const {record_delimiter} = this.options; + const recordDelimiterLength = record_delimiter.length; + loop1: for(let i = 0; i < recordDelimiterLength; i++){ + const rd = record_delimiter[i]; + const rdLength = rd.length; + if(rd[0] !== chr){ + continue; + } + for(let j = 1; j < rdLength; j++){ + if(rd[j] !== buf[pos+j]){ + continue loop1; + } + } + return rd.length; } - for(let j = 1; j < rdLength; j++){ - if(rd[j] !== buf[pos+j]){ - continue loop1; + return 0; + }, + __isEscape: function(buf, pos, chr){ + const {escape} = this.options; + if(escape === null) return false; + const l = escape.length; + if(escape[0] === chr){ + for(let i = 0; i < l; i++){ + if(escape[i] !== buf[pos+i]){ + return false; + } } + return true; } - return rd.length; - } - return 0; - } - __isEscape(buf, pos, chr){ - const {escape} = this.options; - if(escape === null) return false; - const l = escape.length; - if(escape[0] === chr){ + return false; + }, + __isQuote: function(buf, pos){ + const {quote} = this.options; + if(quote === null) return false; + const l = quote.length; for(let i = 0; i < l; i++){ - if(escape[i] !== buf[pos+i]){ + if(quote[i] !== buf[pos+i]){ return false; } } return true; - } - return false; - } - __isQuote(buf, pos){ - const {quote} = this.options; - if(quote === null) return false; - const l = quote.length; - for(let i = 0; i < l; i++){ - if(quote[i] !== buf[pos+i]){ - return false; - } - } - return true; - } - __autoDiscoverRecordDelimiter(buf, pos){ - const {encoding} = this.options; - const chr = buf[pos]; - if(chr === cr){ - if(buf[pos+1] === nl){ - this.options.record_delimiter.push(Buffer.from('\r\n', encoding)); - this.state.recordDelimiterMaxLength = 2; - return 2; - }else { - this.options.record_delimiter.push(Buffer.from('\r', encoding)); + }, + __autoDiscoverRecordDelimiter: function(buf, pos){ + const {encoding} = this.options; + const chr = buf[pos]; + if(chr === cr){ + if(buf[pos+1] === nl){ + this.options.record_delimiter.push(Buffer.from('\r\n', encoding)); + this.state.recordDelimiterMaxLength = 2; + return 2; + }else { + this.options.record_delimiter.push(Buffer.from('\r', encoding)); + this.state.recordDelimiterMaxLength = 1; + return 1; + } + }else if(chr === nl){ + this.options.record_delimiter.push(Buffer.from('\n', encoding)); this.state.recordDelimiterMaxLength = 1; return 1; } - }else if(chr === nl){ - this.options.record_delimiter.push(Buffer.from('\n', encoding)); - this.state.recordDelimiterMaxLength = 1; - return 1; - } - return 0; - } - __error(msg){ - const {encoding, raw, skip_records_with_error} = this.options; - const err = typeof msg === 'string' ? new Error(msg) : msg; - if(skip_records_with_error){ - this.state.recordHasError = true; - this.emit('skip', err, raw ? this.state.rawBuffer.toString(encoding) : undefined); - return undefined; - }else { - return err; + return 0; + }, + __error: function(msg){ + const {encoding, raw, skip_records_with_error} = this.options; + const err = typeof msg === 'string' ? new Error(msg) : msg; + if(skip_records_with_error){ + this.state.recordHasError = true; + if(this.options.on_skip !== undefined){ + this.options.on_skip(err, raw ? this.state.rawBuffer.toString(encoding) : undefined); + } + // this.emit('skip', err, raw ? this.state.rawBuffer.toString(encoding) : undefined); + return undefined; + }else { + return err; + } + }, + __infoDataSet: function(){ + return { + ...this.info, + columns: this.options.columns + }; + }, + __infoRecord: function(){ + const {columns, raw, encoding} = this.options; + return { + ...this.__infoDataSet(), + error: this.state.error, + header: columns === true, + index: this.state.record.length, + raw: raw ? this.state.rawBuffer.toString(encoding) : undefined + }; + }, + __infoField: function(){ + const {columns} = this.options; + const isColumns = Array.isArray(columns); + return { + ...this.__infoRecord(), + column: isColumns === true ? + (columns.length > this.state.record.length ? + columns[this.state.record.length].name : + null + ) : + this.state.record.length, + quoting: this.state.wasQuoting, + }; } - } - __infoDataSet(){ - return { - ...this.info, - columns: this.options.columns + }; + }; + + class Parser extends Transform { + constructor(opts = {}){ + super({...{readableObjectMode: true}, ...opts, encoding: null}); + this.api = transform(opts); + this.api.options.on_skip = (err, chunk) => { + this.emit('skip', err, chunk); }; + // Backward compatibility + this.state = this.api.state; + this.options = this.api.options; + this.info = this.api.info; } - __infoRecord(){ - const {columns, raw, encoding} = this.options; - return { - ...this.__infoDataSet(), - error: this.state.error, - header: columns === true, - index: this.state.record.length, - raw: raw ? this.state.rawBuffer.toString(encoding) : undefined - }; + // Implementation of `Transform._transform` + _transform(buf, encoding, callback){ + if(this.state.stop === true){ + return; + } + const err = this.api.parse(buf, false, (record) => { + this.push.call(this, record); + }, () => { + this.push.call(this, null); + }); + if(err !== undefined){ + this.state.stop = true; + } + callback(err); } - __infoField(){ - const {columns} = this.options; - const isColumns = Array.isArray(columns); - return { - ...this.__infoRecord(), - column: isColumns === true ? - (columns.length > this.state.record.length ? - columns[this.state.record.length].name : - null - ) : - this.state.record.length, - quoting: this.state.wasQuoting, - }; + // Implementation of `Transform._flush` + _flush(callback){ + if(this.state.stop === true){ + return; + } + const err = this.api.parse(undefined, true, (record) => { + this.push.call(this, record); + }, () => { + this.push.call(this, null); + }); + callback(err); } } @@ -6245,7 +6276,7 @@ var csv_parse = (function (exports) { const type = typeof argument; if(data === undefined && (typeof argument === 'string' || isBuffer(argument))){ data = argument; - }else if(options === undefined && isObject(argument)){ + }else if(options === undefined && is_object(argument)){ options = argument; }else if(callback === undefined && type === 'function'){ callback = argument; @@ -6270,10 +6301,10 @@ var csv_parse = (function (exports) { } }); parser.on('error', function(err){ - callback(err, undefined, parser.__infoDataSet()); + callback(err, undefined, parser.api.__infoDataSet()); }); parser.on('end', function(){ - callback(undefined, records, parser.__infoDataSet()); + callback(undefined, records, parser.api.__infoDataSet()); }); } if(data !== undefined){ diff --git a/packages/csv-parse/dist/iife/sync.js b/packages/csv-parse/dist/iife/sync.js index ac98288b9..8aefdf38e 100644 --- a/packages/csv-parse/dist/iife/sync.js +++ b/packages/csv-parse/dist/iife/sync.js @@ -202,7 +202,7 @@ var csv_parse_sync = (function (exports) { var toString = {}.toString; - var isArray$1 = Array.isArray || function (arr) { + var isArray = Array.isArray || function (arr) { return toString.call(arr) == '[object Array]'; }; @@ -470,7 +470,7 @@ var csv_parse_sync = (function (exports) { return fromArrayLike(that, obj) } - if (obj.type === 'Buffer' && isArray$1(obj.data)) { + if (obj.type === 'Buffer' && isArray(obj.data)) { return fromArrayLike(that, obj.data) } } @@ -535,7 +535,7 @@ var csv_parse_sync = (function (exports) { }; Buffer.concat = function concat (list, length) { - if (!isArray$1(list)) { + if (!isArray(list)) { throw new TypeError('"list" argument must be an Array of Buffers') } @@ -1966,3007 +1966,53 @@ var csv_parse_sync = (function (exports) { return typeof obj.readFloatLE === 'function' && typeof obj.slice === 'function' && isFastBuffer(obj.slice(0, 0)) } - var domain; - - // This constructor is used to store event handlers. Instantiating this is - // faster than explicitly calling `Object.create(null)` to get a "clean" empty - // object (tested with v8 v4.9). - function EventHandlers() {} - EventHandlers.prototype = Object.create(null); - - function EventEmitter() { - EventEmitter.init.call(this); - } - - // nodejs oddity - // require('events') === require('events').EventEmitter - EventEmitter.EventEmitter = EventEmitter; - - EventEmitter.usingDomains = false; - - EventEmitter.prototype.domain = undefined; - EventEmitter.prototype._events = undefined; - EventEmitter.prototype._maxListeners = undefined; - - // By default EventEmitters will print a warning if more than 10 listeners are - // added to it. This is a useful default which helps finding memory leaks. - EventEmitter.defaultMaxListeners = 10; - - EventEmitter.init = function() { - this.domain = null; - if (EventEmitter.usingDomains) { - // if there is an active domain, then attach to it. - if (domain.active ) ; - } - - if (!this._events || this._events === Object.getPrototypeOf(this)._events) { - this._events = new EventHandlers(); - this._eventsCount = 0; - } - - this._maxListeners = this._maxListeners || undefined; - }; - - // Obviously not all Emitters should be limited to 10. This function allows - // that to be increased. Set to zero for unlimited. - EventEmitter.prototype.setMaxListeners = function setMaxListeners(n) { - if (typeof n !== 'number' || n < 0 || isNaN(n)) - throw new TypeError('"n" argument must be a positive number'); - this._maxListeners = n; - return this; - }; - - function $getMaxListeners(that) { - if (that._maxListeners === undefined) - return EventEmitter.defaultMaxListeners; - return that._maxListeners; - } - - EventEmitter.prototype.getMaxListeners = function getMaxListeners() { - return $getMaxListeners(this); - }; - - // These standalone emit* functions are used to optimize calling of event - // handlers for fast cases because emit() itself often has a variable number of - // arguments and can be deoptimized because of that. These functions always have - // the same number of arguments and thus do not get deoptimized, so the code - // inside them can execute faster. - function emitNone(handler, isFn, self) { - if (isFn) - handler.call(self); - else { - var len = handler.length; - var listeners = arrayClone(handler, len); - for (var i = 0; i < len; ++i) - listeners[i].call(self); - } - } - function emitOne(handler, isFn, self, arg1) { - if (isFn) - handler.call(self, arg1); - else { - var len = handler.length; - var listeners = arrayClone(handler, len); - for (var i = 0; i < len; ++i) - listeners[i].call(self, arg1); - } - } - function emitTwo(handler, isFn, self, arg1, arg2) { - if (isFn) - handler.call(self, arg1, arg2); - else { - var len = handler.length; - var listeners = arrayClone(handler, len); - for (var i = 0; i < len; ++i) - listeners[i].call(self, arg1, arg2); - } - } - function emitThree(handler, isFn, self, arg1, arg2, arg3) { - if (isFn) - handler.call(self, arg1, arg2, arg3); - else { - var len = handler.length; - var listeners = arrayClone(handler, len); - for (var i = 0; i < len; ++i) - listeners[i].call(self, arg1, arg2, arg3); - } - } - - function emitMany(handler, isFn, self, args) { - if (isFn) - handler.apply(self, args); - else { - var len = handler.length; - var listeners = arrayClone(handler, len); - for (var i = 0; i < len; ++i) - listeners[i].apply(self, args); - } - } - - EventEmitter.prototype.emit = function emit(type) { - var er, handler, len, args, i, events, domain; - var doError = (type === 'error'); - - events = this._events; - if (events) - doError = (doError && events.error == null); - else if (!doError) - return false; - - domain = this.domain; - - // If there is no 'error' event listener then throw. - if (doError) { - er = arguments[1]; - if (domain) { - if (!er) - er = new Error('Uncaught, unspecified "error" event'); - er.domainEmitter = this; - er.domain = domain; - er.domainThrown = false; - domain.emit('error', er); - } else if (er instanceof Error) { - throw er; // Unhandled 'error' event - } else { - // At least give some kind of context to the user - var err = new Error('Uncaught, unspecified "error" event. (' + er + ')'); - err.context = er; - throw err; - } - return false; - } - - handler = events[type]; - - if (!handler) - return false; - - var isFn = typeof handler === 'function'; - len = arguments.length; - switch (len) { - // fast cases - case 1: - emitNone(handler, isFn, this); - break; - case 2: - emitOne(handler, isFn, this, arguments[1]); - break; - case 3: - emitTwo(handler, isFn, this, arguments[1], arguments[2]); - break; - case 4: - emitThree(handler, isFn, this, arguments[1], arguments[2], arguments[3]); - break; - // slower - default: - args = new Array(len - 1); - for (i = 1; i < len; i++) - args[i - 1] = arguments[i]; - emitMany(handler, isFn, this, args); - } - - return true; - }; - - function _addListener(target, type, listener, prepend) { - var m; - var events; - var existing; - - if (typeof listener !== 'function') - throw new TypeError('"listener" argument must be a function'); - - events = target._events; - if (!events) { - events = target._events = new EventHandlers(); - target._eventsCount = 0; - } else { - // To avoid recursion in the case that type === "newListener"! Before - // adding it to the listeners, first emit "newListener". - if (events.newListener) { - target.emit('newListener', type, - listener.listener ? listener.listener : listener); - - // Re-assign `events` because a newListener handler could have caused the - // this._events to be assigned to a new object - events = target._events; - } - existing = events[type]; - } - - if (!existing) { - // Optimize the case of one listener. Don't need the extra array object. - existing = events[type] = listener; - ++target._eventsCount; - } else { - if (typeof existing === 'function') { - // Adding the second element, need to change to array. - existing = events[type] = prepend ? [listener, existing] : - [existing, listener]; - } else { - // If we've already got an array, just append. - if (prepend) { - existing.unshift(listener); - } else { - existing.push(listener); - } - } - - // Check for listener leak - if (!existing.warned) { - m = $getMaxListeners(target); - if (m && m > 0 && existing.length > m) { - existing.warned = true; - var w = new Error('Possible EventEmitter memory leak detected. ' + - existing.length + ' ' + type + ' listeners added. ' + - 'Use emitter.setMaxListeners() to increase limit'); - w.name = 'MaxListenersExceededWarning'; - w.emitter = target; - w.type = type; - w.count = existing.length; - emitWarning(w); - } - } - } - - return target; - } - function emitWarning(e) { - typeof console.warn === 'function' ? console.warn(e) : console.log(e); - } - EventEmitter.prototype.addListener = function addListener(type, listener) { - return _addListener(this, type, listener, false); - }; - - EventEmitter.prototype.on = EventEmitter.prototype.addListener; - - EventEmitter.prototype.prependListener = - function prependListener(type, listener) { - return _addListener(this, type, listener, true); - }; - - function _onceWrap(target, type, listener) { - var fired = false; - function g() { - target.removeListener(type, g); - if (!fired) { - fired = true; - listener.apply(target, arguments); - } - } - g.listener = listener; - return g; - } - - EventEmitter.prototype.once = function once(type, listener) { - if (typeof listener !== 'function') - throw new TypeError('"listener" argument must be a function'); - this.on(type, _onceWrap(this, type, listener)); - return this; - }; - - EventEmitter.prototype.prependOnceListener = - function prependOnceListener(type, listener) { - if (typeof listener !== 'function') - throw new TypeError('"listener" argument must be a function'); - this.prependListener(type, _onceWrap(this, type, listener)); - return this; - }; - - // emits a 'removeListener' event iff the listener was removed - EventEmitter.prototype.removeListener = - function removeListener(type, listener) { - var list, events, position, i, originalListener; - - if (typeof listener !== 'function') - throw new TypeError('"listener" argument must be a function'); - - events = this._events; - if (!events) - return this; - - list = events[type]; - if (!list) - return this; - - if (list === listener || (list.listener && list.listener === listener)) { - if (--this._eventsCount === 0) - this._events = new EventHandlers(); - else { - delete events[type]; - if (events.removeListener) - this.emit('removeListener', type, list.listener || listener); - } - } else if (typeof list !== 'function') { - position = -1; - - for (i = list.length; i-- > 0;) { - if (list[i] === listener || - (list[i].listener && list[i].listener === listener)) { - originalListener = list[i].listener; - position = i; - break; - } - } - - if (position < 0) - return this; - - if (list.length === 1) { - list[0] = undefined; - if (--this._eventsCount === 0) { - this._events = new EventHandlers(); - return this; - } else { - delete events[type]; - } - } else { - spliceOne(list, position); - } - - if (events.removeListener) - this.emit('removeListener', type, originalListener || listener); - } - - return this; - }; - - EventEmitter.prototype.removeAllListeners = - function removeAllListeners(type) { - var listeners, events; - - events = this._events; - if (!events) - return this; - - // not listening for removeListener, no need to emit - if (!events.removeListener) { - if (arguments.length === 0) { - this._events = new EventHandlers(); - this._eventsCount = 0; - } else if (events[type]) { - if (--this._eventsCount === 0) - this._events = new EventHandlers(); - else - delete events[type]; - } - return this; - } - - // emit removeListener for all listeners on all events - if (arguments.length === 0) { - var keys = Object.keys(events); - for (var i = 0, key; i < keys.length; ++i) { - key = keys[i]; - if (key === 'removeListener') continue; - this.removeAllListeners(key); - } - this.removeAllListeners('removeListener'); - this._events = new EventHandlers(); - this._eventsCount = 0; - return this; - } - - listeners = events[type]; - - if (typeof listeners === 'function') { - this.removeListener(type, listeners); - } else if (listeners) { - // LIFO order - do { - this.removeListener(type, listeners[listeners.length - 1]); - } while (listeners[0]); - } - - return this; - }; - - EventEmitter.prototype.listeners = function listeners(type) { - var evlistener; - var ret; - var events = this._events; - - if (!events) - ret = []; - else { - evlistener = events[type]; - if (!evlistener) - ret = []; - else if (typeof evlistener === 'function') - ret = [evlistener.listener || evlistener]; - else - ret = unwrapListeners(evlistener); - } - - return ret; - }; - - EventEmitter.listenerCount = function(emitter, type) { - if (typeof emitter.listenerCount === 'function') { - return emitter.listenerCount(type); - } else { - return listenerCount$1.call(emitter, type); - } - }; - - EventEmitter.prototype.listenerCount = listenerCount$1; - function listenerCount$1(type) { - var events = this._events; - - if (events) { - var evlistener = events[type]; - - if (typeof evlistener === 'function') { - return 1; - } else if (evlistener) { - return evlistener.length; - } - } - - return 0; - } - - EventEmitter.prototype.eventNames = function eventNames() { - return this._eventsCount > 0 ? Reflect.ownKeys(this._events) : []; - }; - - // About 1.5x faster than the two-arg version of Array#splice(). - function spliceOne(list, index) { - for (var i = index, k = i + 1, n = list.length; k < n; i += 1, k += 1) - list[i] = list[k]; - list.pop(); - } - - function arrayClone(arr, i) { - var copy = new Array(i); - while (i--) - copy[i] = arr[i]; - return copy; - } - - function unwrapListeners(arr) { - var ret = new Array(arr.length); - for (var i = 0; i < ret.length; ++i) { - ret[i] = arr[i].listener || arr[i]; - } - return ret; - } - - // shim for using process in browser - // based off https://github.com/defunctzombie/node-process/blob/master/browser.js - - function defaultSetTimout() { - throw new Error('setTimeout has not been defined'); - } - function defaultClearTimeout () { - throw new Error('clearTimeout has not been defined'); - } - var cachedSetTimeout = defaultSetTimout; - var cachedClearTimeout = defaultClearTimeout; - if (typeof global$1.setTimeout === 'function') { - cachedSetTimeout = setTimeout; - } - if (typeof global$1.clearTimeout === 'function') { - cachedClearTimeout = clearTimeout; - } - - function runTimeout(fun) { - if (cachedSetTimeout === setTimeout) { - //normal enviroments in sane situations - return setTimeout(fun, 0); - } - // if setTimeout wasn't available but was latter defined - if ((cachedSetTimeout === defaultSetTimout || !cachedSetTimeout) && setTimeout) { - cachedSetTimeout = setTimeout; - return setTimeout(fun, 0); - } - try { - // when when somebody has screwed with setTimeout but no I.E. maddness - return cachedSetTimeout(fun, 0); - } catch(e){ - try { - // When we are in I.E. but the script has been evaled so I.E. doesn't trust the global object when called normally - return cachedSetTimeout.call(null, fun, 0); - } catch(e){ - // same as above but when it's a version of I.E. that must have the global object for 'this', hopfully our context correct otherwise it will throw a global error - return cachedSetTimeout.call(this, fun, 0); - } - } - - - } - function runClearTimeout(marker) { - if (cachedClearTimeout === clearTimeout) { - //normal enviroments in sane situations - return clearTimeout(marker); - } - // if clearTimeout wasn't available but was latter defined - if ((cachedClearTimeout === defaultClearTimeout || !cachedClearTimeout) && clearTimeout) { - cachedClearTimeout = clearTimeout; - return clearTimeout(marker); - } - try { - // when when somebody has screwed with setTimeout but no I.E. maddness - return cachedClearTimeout(marker); - } catch (e){ - try { - // When we are in I.E. but the script has been evaled so I.E. doesn't trust the global object when called normally - return cachedClearTimeout.call(null, marker); - } catch (e){ - // same as above but when it's a version of I.E. that must have the global object for 'this', hopfully our context correct otherwise it will throw a global error. - // Some versions of I.E. have different rules for clearTimeout vs setTimeout - return cachedClearTimeout.call(this, marker); - } - } - - - - } - var queue = []; - var draining = false; - var currentQueue; - var queueIndex = -1; - - function cleanUpNextTick() { - if (!draining || !currentQueue) { - return; - } - draining = false; - if (currentQueue.length) { - queue = currentQueue.concat(queue); - } else { - queueIndex = -1; - } - if (queue.length) { - drainQueue(); - } - } - - function drainQueue() { - if (draining) { - return; - } - var timeout = runTimeout(cleanUpNextTick); - draining = true; - - var len = queue.length; - while(len) { - currentQueue = queue; - queue = []; - while (++queueIndex < len) { - if (currentQueue) { - currentQueue[queueIndex].run(); - } - } - queueIndex = -1; - len = queue.length; - } - currentQueue = null; - draining = false; - runClearTimeout(timeout); - } - function nextTick(fun) { - var args = new Array(arguments.length - 1); - if (arguments.length > 1) { - for (var i = 1; i < arguments.length; i++) { - args[i - 1] = arguments[i]; - } - } - queue.push(new Item(fun, args)); - if (queue.length === 1 && !draining) { - runTimeout(drainQueue); - } - } - // v8 likes predictible objects - function Item(fun, array) { - this.fun = fun; - this.array = array; - } - Item.prototype.run = function () { - this.fun.apply(null, this.array); - }; - - // from https://github.com/kumavis/browser-process-hrtime/blob/master/index.js - var performance = global$1.performance || {}; - performance.now || - performance.mozNow || - performance.msNow || - performance.oNow || - performance.webkitNow || - function(){ return (new Date()).getTime() }; - - var inherits; - if (typeof Object.create === 'function'){ - inherits = function inherits(ctor, superCtor) { - // implementation from standard node.js 'util' module - ctor.super_ = superCtor; - ctor.prototype = Object.create(superCtor.prototype, { - constructor: { - value: ctor, - enumerable: false, - writable: true, - configurable: true - } - }); - }; - } else { - inherits = function inherits(ctor, superCtor) { - ctor.super_ = superCtor; - var TempCtor = function () {}; - TempCtor.prototype = superCtor.prototype; - ctor.prototype = new TempCtor(); - ctor.prototype.constructor = ctor; - }; - } - var inherits$1 = inherits; - - var formatRegExp = /%[sdj%]/g; - function format(f) { - if (!isString(f)) { - var objects = []; - for (var i = 0; i < arguments.length; i++) { - objects.push(inspect(arguments[i])); - } - return objects.join(' '); - } - - var i = 1; - var args = arguments; - var len = args.length; - var str = String(f).replace(formatRegExp, function(x) { - if (x === '%%') return '%'; - if (i >= len) return x; - switch (x) { - case '%s': return String(args[i++]); - case '%d': return Number(args[i++]); - case '%j': - try { - return JSON.stringify(args[i++]); - } catch (_) { - return '[Circular]'; - } - default: - return x; - } - }); - for (var x = args[i]; i < len; x = args[++i]) { - if (isNull(x) || !isObject$1(x)) { - str += ' ' + x; - } else { - str += ' ' + inspect(x); - } - } - return str; - } - - // Mark that a method should not be used. - // Returns a modified function which warns once by default. - // If --no-deprecation is set, then it is a no-op. - function deprecate(fn, msg) { - // Allow for deprecating things in the process of starting up. - if (isUndefined(global$1.process)) { - return function() { - return deprecate(fn, msg).apply(this, arguments); - }; - } - - var warned = false; - function deprecated() { - if (!warned) { - { - console.error(msg); - } - warned = true; - } - return fn.apply(this, arguments); - } - - return deprecated; - } - - var debugs = {}; - var debugEnviron; - function debuglog(set) { - if (isUndefined(debugEnviron)) - debugEnviron = ''; - set = set.toUpperCase(); - if (!debugs[set]) { - if (new RegExp('\\b' + set + '\\b', 'i').test(debugEnviron)) { - var pid = 0; - debugs[set] = function() { - var msg = format.apply(null, arguments); - console.error('%s %d: %s', set, pid, msg); - }; - } else { - debugs[set] = function() {}; - } - } - return debugs[set]; - } - - /** - * Echos the value of a value. Trys to print the value out - * in the best way possible given the different types. - * - * @param {Object} obj The object to print out. - * @param {Object} opts Optional options object that alters the output. - */ - /* legacy: obj, showHidden, depth, colors*/ - function inspect(obj, opts) { - // default options - var ctx = { - seen: [], - stylize: stylizeNoColor - }; - // legacy... - if (arguments.length >= 3) ctx.depth = arguments[2]; - if (arguments.length >= 4) ctx.colors = arguments[3]; - if (isBoolean(opts)) { - // legacy... - ctx.showHidden = opts; - } else if (opts) { - // got an "options" object - _extend(ctx, opts); - } - // set default options - if (isUndefined(ctx.showHidden)) ctx.showHidden = false; - if (isUndefined(ctx.depth)) ctx.depth = 2; - if (isUndefined(ctx.colors)) ctx.colors = false; - if (isUndefined(ctx.customInspect)) ctx.customInspect = true; - if (ctx.colors) ctx.stylize = stylizeWithColor; - return formatValue(ctx, obj, ctx.depth); - } - - // http://en.wikipedia.org/wiki/ANSI_escape_code#graphics - inspect.colors = { - 'bold' : [1, 22], - 'italic' : [3, 23], - 'underline' : [4, 24], - 'inverse' : [7, 27], - 'white' : [37, 39], - 'grey' : [90, 39], - 'black' : [30, 39], - 'blue' : [34, 39], - 'cyan' : [36, 39], - 'green' : [32, 39], - 'magenta' : [35, 39], - 'red' : [31, 39], - 'yellow' : [33, 39] - }; - - // Don't use 'blue' not visible on cmd.exe - inspect.styles = { - 'special': 'cyan', - 'number': 'yellow', - 'boolean': 'yellow', - 'undefined': 'grey', - 'null': 'bold', - 'string': 'green', - 'date': 'magenta', - // "name": intentionally not styling - 'regexp': 'red' - }; - - - function stylizeWithColor(str, styleType) { - var style = inspect.styles[styleType]; - - if (style) { - return '\u001b[' + inspect.colors[style][0] + 'm' + str + - '\u001b[' + inspect.colors[style][1] + 'm'; - } else { - return str; - } - } - - - function stylizeNoColor(str, styleType) { - return str; - } - - - function arrayToHash(array) { - var hash = {}; - - array.forEach(function(val, idx) { - hash[val] = true; - }); - - return hash; - } - - - function formatValue(ctx, value, recurseTimes) { - // Provide a hook for user-specified inspect functions. - // Check that value is an object with an inspect function on it - if (ctx.customInspect && - value && - isFunction(value.inspect) && - // Filter out the util module, it's inspect function is special - value.inspect !== inspect && - // Also filter out any prototype objects using the circular check. - !(value.constructor && value.constructor.prototype === value)) { - var ret = value.inspect(recurseTimes, ctx); - if (!isString(ret)) { - ret = formatValue(ctx, ret, recurseTimes); - } - return ret; - } - - // Primitive types cannot have properties - var primitive = formatPrimitive(ctx, value); - if (primitive) { - return primitive; - } - - // Look up the keys of the object. - var keys = Object.keys(value); - var visibleKeys = arrayToHash(keys); - - if (ctx.showHidden) { - keys = Object.getOwnPropertyNames(value); - } - - // IE doesn't make error fields non-enumerable - // http://msdn.microsoft.com/en-us/library/ie/dww52sbt(v=vs.94).aspx - if (isError(value) - && (keys.indexOf('message') >= 0 || keys.indexOf('description') >= 0)) { - return formatError(value); - } - - // Some type of object without properties can be shortcutted. - if (keys.length === 0) { - if (isFunction(value)) { - var name = value.name ? ': ' + value.name : ''; - return ctx.stylize('[Function' + name + ']', 'special'); - } - if (isRegExp(value)) { - return ctx.stylize(RegExp.prototype.toString.call(value), 'regexp'); - } - if (isDate(value)) { - return ctx.stylize(Date.prototype.toString.call(value), 'date'); - } - if (isError(value)) { - return formatError(value); - } - } - - var base = '', array = false, braces = ['{', '}']; - - // Make Array say that they are Array - if (isArray(value)) { - array = true; - braces = ['[', ']']; - } - - // Make functions say that they are functions - if (isFunction(value)) { - var n = value.name ? ': ' + value.name : ''; - base = ' [Function' + n + ']'; - } - - // Make RegExps say that they are RegExps - if (isRegExp(value)) { - base = ' ' + RegExp.prototype.toString.call(value); - } - - // Make dates with properties first say the date - if (isDate(value)) { - base = ' ' + Date.prototype.toUTCString.call(value); - } - - // Make error with message first say the error - if (isError(value)) { - base = ' ' + formatError(value); - } - - if (keys.length === 0 && (!array || value.length == 0)) { - return braces[0] + base + braces[1]; - } - - if (recurseTimes < 0) { - if (isRegExp(value)) { - return ctx.stylize(RegExp.prototype.toString.call(value), 'regexp'); - } else { - return ctx.stylize('[Object]', 'special'); - } - } - - ctx.seen.push(value); - - var output; - if (array) { - output = formatArray(ctx, value, recurseTimes, visibleKeys, keys); - } else { - output = keys.map(function(key) { - return formatProperty(ctx, value, recurseTimes, visibleKeys, key, array); - }); - } - - ctx.seen.pop(); - - return reduceToSingleString(output, base, braces); - } - - - function formatPrimitive(ctx, value) { - if (isUndefined(value)) - return ctx.stylize('undefined', 'undefined'); - if (isString(value)) { - var simple = '\'' + JSON.stringify(value).replace(/^"|"$/g, '') - .replace(/'/g, "\\'") - .replace(/\\"/g, '"') + '\''; - return ctx.stylize(simple, 'string'); - } - if (isNumber(value)) - return ctx.stylize('' + value, 'number'); - if (isBoolean(value)) - return ctx.stylize('' + value, 'boolean'); - // For some reason typeof null is "object", so special case here. - if (isNull(value)) - return ctx.stylize('null', 'null'); - } - - - function formatError(value) { - return '[' + Error.prototype.toString.call(value) + ']'; - } - - - function formatArray(ctx, value, recurseTimes, visibleKeys, keys) { - var output = []; - for (var i = 0, l = value.length; i < l; ++i) { - if (hasOwnProperty(value, String(i))) { - output.push(formatProperty(ctx, value, recurseTimes, visibleKeys, - String(i), true)); - } else { - output.push(''); - } - } - keys.forEach(function(key) { - if (!key.match(/^\d+$/)) { - output.push(formatProperty(ctx, value, recurseTimes, visibleKeys, - key, true)); - } - }); - return output; - } - - - function formatProperty(ctx, value, recurseTimes, visibleKeys, key, array) { - var name, str, desc; - desc = Object.getOwnPropertyDescriptor(value, key) || { value: value[key] }; - if (desc.get) { - if (desc.set) { - str = ctx.stylize('[Getter/Setter]', 'special'); - } else { - str = ctx.stylize('[Getter]', 'special'); - } - } else { - if (desc.set) { - str = ctx.stylize('[Setter]', 'special'); - } - } - if (!hasOwnProperty(visibleKeys, key)) { - name = '[' + key + ']'; - } - if (!str) { - if (ctx.seen.indexOf(desc.value) < 0) { - if (isNull(recurseTimes)) { - str = formatValue(ctx, desc.value, null); - } else { - str = formatValue(ctx, desc.value, recurseTimes - 1); - } - if (str.indexOf('\n') > -1) { - if (array) { - str = str.split('\n').map(function(line) { - return ' ' + line; - }).join('\n').substr(2); - } else { - str = '\n' + str.split('\n').map(function(line) { - return ' ' + line; - }).join('\n'); - } - } - } else { - str = ctx.stylize('[Circular]', 'special'); - } - } - if (isUndefined(name)) { - if (array && key.match(/^\d+$/)) { - return str; - } - name = JSON.stringify('' + key); - if (name.match(/^"([a-zA-Z_][a-zA-Z_0-9]*)"$/)) { - name = name.substr(1, name.length - 2); - name = ctx.stylize(name, 'name'); - } else { - name = name.replace(/'/g, "\\'") - .replace(/\\"/g, '"') - .replace(/(^"|"$)/g, "'"); - name = ctx.stylize(name, 'string'); - } - } - - return name + ': ' + str; - } - - - function reduceToSingleString(output, base, braces) { - var length = output.reduce(function(prev, cur) { - if (cur.indexOf('\n') >= 0) ; - return prev + cur.replace(/\u001b\[\d\d?m/g, '').length + 1; - }, 0); - - if (length > 60) { - return braces[0] + - (base === '' ? '' : base + '\n ') + - ' ' + - output.join(',\n ') + - ' ' + - braces[1]; - } - - return braces[0] + base + ' ' + output.join(', ') + ' ' + braces[1]; - } - - - // NOTE: These type checking functions intentionally don't use `instanceof` - // because it is fragile and can be easily faked with `Object.create()`. - function isArray(ar) { - return Array.isArray(ar); - } - - function isBoolean(arg) { - return typeof arg === 'boolean'; - } - - function isNull(arg) { - return arg === null; - } - - function isNumber(arg) { - return typeof arg === 'number'; - } - - function isString(arg) { - return typeof arg === 'string'; - } - - function isUndefined(arg) { - return arg === void 0; - } - - function isRegExp(re) { - return isObject$1(re) && objectToString(re) === '[object RegExp]'; - } - - function isObject$1(arg) { - return typeof arg === 'object' && arg !== null; - } - - function isDate(d) { - return isObject$1(d) && objectToString(d) === '[object Date]'; - } - - function isError(e) { - return isObject$1(e) && - (objectToString(e) === '[object Error]' || e instanceof Error); - } - - function isFunction(arg) { - return typeof arg === 'function'; - } - - function objectToString(o) { - return Object.prototype.toString.call(o); - } - - function _extend(origin, add) { - // Don't do anything if add isn't an object - if (!add || !isObject$1(add)) return origin; - - var keys = Object.keys(add); - var i = keys.length; - while (i--) { - origin[keys[i]] = add[keys[i]]; - } - return origin; - } - function hasOwnProperty(obj, prop) { - return Object.prototype.hasOwnProperty.call(obj, prop); - } - - function BufferList() { - this.head = null; - this.tail = null; - this.length = 0; - } - - BufferList.prototype.push = function (v) { - var entry = { data: v, next: null }; - if (this.length > 0) this.tail.next = entry;else this.head = entry; - this.tail = entry; - ++this.length; - }; - - BufferList.prototype.unshift = function (v) { - var entry = { data: v, next: this.head }; - if (this.length === 0) this.tail = entry; - this.head = entry; - ++this.length; - }; - - BufferList.prototype.shift = function () { - if (this.length === 0) return; - var ret = this.head.data; - if (this.length === 1) this.head = this.tail = null;else this.head = this.head.next; - --this.length; - return ret; - }; - - BufferList.prototype.clear = function () { - this.head = this.tail = null; - this.length = 0; - }; - - BufferList.prototype.join = function (s) { - if (this.length === 0) return ''; - var p = this.head; - var ret = '' + p.data; - while (p = p.next) { - ret += s + p.data; - }return ret; - }; - - BufferList.prototype.concat = function (n) { - if (this.length === 0) return Buffer.alloc(0); - if (this.length === 1) return this.head.data; - var ret = Buffer.allocUnsafe(n >>> 0); - var p = this.head; - var i = 0; - while (p) { - p.data.copy(ret, i); - i += p.data.length; - p = p.next; - } - return ret; - }; - - // Copyright Joyent, Inc. and other Node contributors. - var isBufferEncoding = Buffer.isEncoding - || function(encoding) { - switch (encoding && encoding.toLowerCase()) { - case 'hex': case 'utf8': case 'utf-8': case 'ascii': case 'binary': case 'base64': case 'ucs2': case 'ucs-2': case 'utf16le': case 'utf-16le': case 'raw': return true; - default: return false; - } - }; - - - function assertEncoding(encoding) { - if (encoding && !isBufferEncoding(encoding)) { - throw new Error('Unknown encoding: ' + encoding); - } - } - - // StringDecoder provides an interface for efficiently splitting a series of - // buffers into a series of JS strings without breaking apart multi-byte - // characters. CESU-8 is handled as part of the UTF-8 encoding. - // - // @TODO Handling all encodings inside a single object makes it very difficult - // to reason about this code, so it should be split up in the future. - // @TODO There should be a utf8-strict encoding that rejects invalid UTF-8 code - // points as used by CESU-8. - function StringDecoder(encoding) { - this.encoding = (encoding || 'utf8').toLowerCase().replace(/[-_]/, ''); - assertEncoding(encoding); - switch (this.encoding) { - case 'utf8': - // CESU-8 represents each of Surrogate Pair by 3-bytes - this.surrogateSize = 3; - break; - case 'ucs2': - case 'utf16le': - // UTF-16 represents each of Surrogate Pair by 2-bytes - this.surrogateSize = 2; - this.detectIncompleteChar = utf16DetectIncompleteChar; - break; - case 'base64': - // Base-64 stores 3 bytes in 4 chars, and pads the remainder. - this.surrogateSize = 3; - this.detectIncompleteChar = base64DetectIncompleteChar; - break; - default: - this.write = passThroughWrite; - return; - } - - // Enough space to store all bytes of a single character. UTF-8 needs 4 - // bytes, but CESU-8 may require up to 6 (3 bytes per surrogate). - this.charBuffer = new Buffer(6); - // Number of bytes received for the current incomplete multi-byte character. - this.charReceived = 0; - // Number of bytes expected for the current incomplete multi-byte character. - this.charLength = 0; - } - - // write decodes the given buffer and returns it as JS string that is - // guaranteed to not contain any partial multi-byte characters. Any partial - // character found at the end of the buffer is buffered up, and will be - // returned when calling write again with the remaining bytes. - // - // Note: Converting a Buffer containing an orphan surrogate to a String - // currently works, but converting a String to a Buffer (via `new Buffer`, or - // Buffer#write) will replace incomplete surrogates with the unicode - // replacement character. See https://codereview.chromium.org/121173009/ . - StringDecoder.prototype.write = function(buffer) { - var charStr = ''; - // if our last write ended with an incomplete multibyte character - while (this.charLength) { - // determine how many remaining bytes this buffer has to offer for this char - var available = (buffer.length >= this.charLength - this.charReceived) ? - this.charLength - this.charReceived : - buffer.length; - - // add the new bytes to the char buffer - buffer.copy(this.charBuffer, this.charReceived, 0, available); - this.charReceived += available; - - if (this.charReceived < this.charLength) { - // still not enough chars in this buffer? wait for more ... - return ''; - } - - // remove bytes belonging to the current character from the buffer - buffer = buffer.slice(available, buffer.length); - - // get the character that was split - charStr = this.charBuffer.slice(0, this.charLength).toString(this.encoding); - - // CESU-8: lead surrogate (D800-DBFF) is also the incomplete character - var charCode = charStr.charCodeAt(charStr.length - 1); - if (charCode >= 0xD800 && charCode <= 0xDBFF) { - this.charLength += this.surrogateSize; - charStr = ''; - continue; - } - this.charReceived = this.charLength = 0; - - // if there are no more bytes in this buffer, just emit our char - if (buffer.length === 0) { - return charStr; - } - break; - } - - // determine and set charLength / charReceived - this.detectIncompleteChar(buffer); - - var end = buffer.length; - if (this.charLength) { - // buffer the incomplete character bytes we got - buffer.copy(this.charBuffer, 0, buffer.length - this.charReceived, end); - end -= this.charReceived; - } - - charStr += buffer.toString(this.encoding, 0, end); - - var end = charStr.length - 1; - var charCode = charStr.charCodeAt(end); - // CESU-8: lead surrogate (D800-DBFF) is also the incomplete character - if (charCode >= 0xD800 && charCode <= 0xDBFF) { - var size = this.surrogateSize; - this.charLength += size; - this.charReceived += size; - this.charBuffer.copy(this.charBuffer, size, 0, size); - buffer.copy(this.charBuffer, 0, 0, size); - return charStr.substring(0, end); - } - - // or just emit the charStr - return charStr; - }; - - // detectIncompleteChar determines if there is an incomplete UTF-8 character at - // the end of the given buffer. If so, it sets this.charLength to the byte - // length that character, and sets this.charReceived to the number of bytes - // that are available for this character. - StringDecoder.prototype.detectIncompleteChar = function(buffer) { - // determine how many bytes we have to check at the end of this buffer - var i = (buffer.length >= 3) ? 3 : buffer.length; - - // Figure out if one of the last i bytes of our buffer announces an - // incomplete char. - for (; i > 0; i--) { - var c = buffer[buffer.length - i]; - - // See http://en.wikipedia.org/wiki/UTF-8#Description - - // 110XXXXX - if (i == 1 && c >> 5 == 0x06) { - this.charLength = 2; - break; - } - - // 1110XXXX - if (i <= 2 && c >> 4 == 0x0E) { - this.charLength = 3; - break; - } - - // 11110XXX - if (i <= 3 && c >> 3 == 0x1E) { - this.charLength = 4; - break; - } - } - this.charReceived = i; - }; - - StringDecoder.prototype.end = function(buffer) { - var res = ''; - if (buffer && buffer.length) - res = this.write(buffer); - - if (this.charReceived) { - var cr = this.charReceived; - var buf = this.charBuffer; - var enc = this.encoding; - res += buf.slice(0, cr).toString(enc); - } - - return res; - }; - - function passThroughWrite(buffer) { - return buffer.toString(this.encoding); - } - - function utf16DetectIncompleteChar(buffer) { - this.charReceived = buffer.length % 2; - this.charLength = this.charReceived ? 2 : 0; - } - - function base64DetectIncompleteChar(buffer) { - this.charReceived = buffer.length % 3; - this.charLength = this.charReceived ? 3 : 0; - } - - Readable.ReadableState = ReadableState; - - var debug = debuglog('stream'); - inherits$1(Readable, EventEmitter); - - function prependListener(emitter, event, fn) { - // Sadly this is not cacheable as some libraries bundle their own - // event emitter implementation with them. - if (typeof emitter.prependListener === 'function') { - return emitter.prependListener(event, fn); - } else { - // This is a hack to make sure that our error handler is attached before any - // userland ones. NEVER DO THIS. This is here only because this code needs - // to continue to work with older versions of Node.js that do not include - // the prependListener() method. The goal is to eventually remove this hack. - if (!emitter._events || !emitter._events[event]) - emitter.on(event, fn); - else if (Array.isArray(emitter._events[event])) - emitter._events[event].unshift(fn); - else - emitter._events[event] = [fn, emitter._events[event]]; - } - } - function listenerCount (emitter, type) { - return emitter.listeners(type).length; - } - function ReadableState(options, stream) { - - options = options || {}; - - // object stream flag. Used to make read(n) ignore n and to - // make all the buffer merging and length checks go away - this.objectMode = !!options.objectMode; - - if (stream instanceof Duplex) this.objectMode = this.objectMode || !!options.readableObjectMode; - - // the point at which it stops calling _read() to fill the buffer - // Note: 0 is a valid value, means "don't call _read preemptively ever" - var hwm = options.highWaterMark; - var defaultHwm = this.objectMode ? 16 : 16 * 1024; - this.highWaterMark = hwm || hwm === 0 ? hwm : defaultHwm; - - // cast to ints. - this.highWaterMark = ~ ~this.highWaterMark; - - // A linked list is used to store data chunks instead of an array because the - // linked list can remove elements from the beginning faster than - // array.shift() - this.buffer = new BufferList(); - this.length = 0; - this.pipes = null; - this.pipesCount = 0; - this.flowing = null; - this.ended = false; - this.endEmitted = false; - this.reading = false; - - // a flag to be able to tell if the onwrite cb is called immediately, - // or on a later tick. We set this to true at first, because any - // actions that shouldn't happen until "later" should generally also - // not happen before the first write call. - this.sync = true; - - // whenever we return null, then we set a flag to say - // that we're awaiting a 'readable' event emission. - this.needReadable = false; - this.emittedReadable = false; - this.readableListening = false; - this.resumeScheduled = false; - - // Crypto is kind of old and crusty. Historically, its default string - // encoding is 'binary' so we have to make this configurable. - // Everything else in the universe uses 'utf8', though. - this.defaultEncoding = options.defaultEncoding || 'utf8'; - - // when piping, we only care about 'readable' events that happen - // after read()ing all the bytes and not getting any pushback. - this.ranOut = false; - - // the number of writers that are awaiting a drain event in .pipe()s - this.awaitDrain = 0; - - // if true, a maybeReadMore has been scheduled - this.readingMore = false; - - this.decoder = null; - this.encoding = null; - if (options.encoding) { - this.decoder = new StringDecoder(options.encoding); - this.encoding = options.encoding; - } - } - function Readable(options) { - - if (!(this instanceof Readable)) return new Readable(options); - - this._readableState = new ReadableState(options, this); - - // legacy - this.readable = true; - - if (options && typeof options.read === 'function') this._read = options.read; - - EventEmitter.call(this); - } - - // Manually shove something into the read() buffer. - // This returns true if the highWaterMark has not been hit yet, - // similar to how Writable.write() returns true if you should - // write() some more. - Readable.prototype.push = function (chunk, encoding) { - var state = this._readableState; - - if (!state.objectMode && typeof chunk === 'string') { - encoding = encoding || state.defaultEncoding; - if (encoding !== state.encoding) { - chunk = Buffer.from(chunk, encoding); - encoding = ''; - } - } - - return readableAddChunk(this, state, chunk, encoding, false); - }; - - // Unshift should *always* be something directly out of read() - Readable.prototype.unshift = function (chunk) { - var state = this._readableState; - return readableAddChunk(this, state, chunk, '', true); - }; - - Readable.prototype.isPaused = function () { - return this._readableState.flowing === false; - }; - - function readableAddChunk(stream, state, chunk, encoding, addToFront) { - var er = chunkInvalid(state, chunk); - if (er) { - stream.emit('error', er); - } else if (chunk === null) { - state.reading = false; - onEofChunk(stream, state); - } else if (state.objectMode || chunk && chunk.length > 0) { - if (state.ended && !addToFront) { - var e = new Error('stream.push() after EOF'); - stream.emit('error', e); - } else if (state.endEmitted && addToFront) { - var _e = new Error('stream.unshift() after end event'); - stream.emit('error', _e); - } else { - var skipAdd; - if (state.decoder && !addToFront && !encoding) { - chunk = state.decoder.write(chunk); - skipAdd = !state.objectMode && chunk.length === 0; - } - - if (!addToFront) state.reading = false; - - // Don't add to the buffer if we've decoded to an empty string chunk and - // we're not in object mode - if (!skipAdd) { - // if we want the data now, just emit it. - if (state.flowing && state.length === 0 && !state.sync) { - stream.emit('data', chunk); - stream.read(0); - } else { - // update the buffer info. - state.length += state.objectMode ? 1 : chunk.length; - if (addToFront) state.buffer.unshift(chunk);else state.buffer.push(chunk); - - if (state.needReadable) emitReadable(stream); - } - } - - maybeReadMore(stream, state); - } - } else if (!addToFront) { - state.reading = false; - } - - return needMoreData(state); - } - - // if it's past the high water mark, we can push in some more. - // Also, if we have no data yet, we can stand some - // more bytes. This is to work around cases where hwm=0, - // such as the repl. Also, if the push() triggered a - // readable event, and the user called read(largeNumber) such that - // needReadable was set, then we ought to push more, so that another - // 'readable' event will be triggered. - function needMoreData(state) { - return !state.ended && (state.needReadable || state.length < state.highWaterMark || state.length === 0); - } - - // backwards compatibility. - Readable.prototype.setEncoding = function (enc) { - this._readableState.decoder = new StringDecoder(enc); - this._readableState.encoding = enc; - return this; - }; - - // Don't raise the hwm > 8MB - var MAX_HWM = 0x800000; - function computeNewHighWaterMark(n) { - if (n >= MAX_HWM) { - n = MAX_HWM; - } else { - // Get the next highest power of 2 to prevent increasing hwm excessively in - // tiny amounts - n--; - n |= n >>> 1; - n |= n >>> 2; - n |= n >>> 4; - n |= n >>> 8; - n |= n >>> 16; - n++; - } - return n; - } - - // This function is designed to be inlinable, so please take care when making - // changes to the function body. - function howMuchToRead(n, state) { - if (n <= 0 || state.length === 0 && state.ended) return 0; - if (state.objectMode) return 1; - if (n !== n) { - // Only flow one buffer at a time - if (state.flowing && state.length) return state.buffer.head.data.length;else return state.length; - } - // If we're asking for more than the current hwm, then raise the hwm. - if (n > state.highWaterMark) state.highWaterMark = computeNewHighWaterMark(n); - if (n <= state.length) return n; - // Don't have enough - if (!state.ended) { - state.needReadable = true; - return 0; - } - return state.length; - } - - // you can override either this method, or the async _read(n) below. - Readable.prototype.read = function (n) { - debug('read', n); - n = parseInt(n, 10); - var state = this._readableState; - var nOrig = n; - - if (n !== 0) state.emittedReadable = false; - - // if we're doing read(0) to trigger a readable event, but we - // already have a bunch of data in the buffer, then just trigger - // the 'readable' event and move on. - if (n === 0 && state.needReadable && (state.length >= state.highWaterMark || state.ended)) { - debug('read: emitReadable', state.length, state.ended); - if (state.length === 0 && state.ended) endReadable(this);else emitReadable(this); - return null; - } - - n = howMuchToRead(n, state); - - // if we've ended, and we're now clear, then finish it up. - if (n === 0 && state.ended) { - if (state.length === 0) endReadable(this); - return null; - } - - // All the actual chunk generation logic needs to be - // *below* the call to _read. The reason is that in certain - // synthetic stream cases, such as passthrough streams, _read - // may be a completely synchronous operation which may change - // the state of the read buffer, providing enough data when - // before there was *not* enough. - // - // So, the steps are: - // 1. Figure out what the state of things will be after we do - // a read from the buffer. - // - // 2. If that resulting state will trigger a _read, then call _read. - // Note that this may be asynchronous, or synchronous. Yes, it is - // deeply ugly to write APIs this way, but that still doesn't mean - // that the Readable class should behave improperly, as streams are - // designed to be sync/async agnostic. - // Take note if the _read call is sync or async (ie, if the read call - // has returned yet), so that we know whether or not it's safe to emit - // 'readable' etc. - // - // 3. Actually pull the requested chunks out of the buffer and return. - - // if we need a readable event, then we need to do some reading. - var doRead = state.needReadable; - debug('need readable', doRead); - - // if we currently have less than the highWaterMark, then also read some - if (state.length === 0 || state.length - n < state.highWaterMark) { - doRead = true; - debug('length less than watermark', doRead); - } - - // however, if we've ended, then there's no point, and if we're already - // reading, then it's unnecessary. - if (state.ended || state.reading) { - doRead = false; - debug('reading or ended', doRead); - } else if (doRead) { - debug('do read'); - state.reading = true; - state.sync = true; - // if the length is currently zero, then we *need* a readable event. - if (state.length === 0) state.needReadable = true; - // call internal read method - this._read(state.highWaterMark); - state.sync = false; - // If _read pushed data synchronously, then `reading` will be false, - // and we need to re-evaluate how much data we can return to the user. - if (!state.reading) n = howMuchToRead(nOrig, state); - } - - var ret; - if (n > 0) ret = fromList(n, state);else ret = null; - - if (ret === null) { - state.needReadable = true; - n = 0; - } else { - state.length -= n; - } - - if (state.length === 0) { - // If we have nothing in the buffer, then we want to know - // as soon as we *do* get something into the buffer. - if (!state.ended) state.needReadable = true; - - // If we tried to read() past the EOF, then emit end on the next tick. - if (nOrig !== n && state.ended) endReadable(this); - } - - if (ret !== null) this.emit('data', ret); - - return ret; - }; - - function chunkInvalid(state, chunk) { - var er = null; - if (!isBuffer(chunk) && typeof chunk !== 'string' && chunk !== null && chunk !== undefined && !state.objectMode) { - er = new TypeError('Invalid non-string/buffer chunk'); - } - return er; - } - - function onEofChunk(stream, state) { - if (state.ended) return; - if (state.decoder) { - var chunk = state.decoder.end(); - if (chunk && chunk.length) { - state.buffer.push(chunk); - state.length += state.objectMode ? 1 : chunk.length; - } - } - state.ended = true; - - // emit 'readable' now to make sure it gets picked up. - emitReadable(stream); - } - - // Don't emit readable right away in sync mode, because this can trigger - // another read() call => stack overflow. This way, it might trigger - // a nextTick recursion warning, but that's not so bad. - function emitReadable(stream) { - var state = stream._readableState; - state.needReadable = false; - if (!state.emittedReadable) { - debug('emitReadable', state.flowing); - state.emittedReadable = true; - if (state.sync) nextTick(emitReadable_, stream);else emitReadable_(stream); - } - } - - function emitReadable_(stream) { - debug('emit readable'); - stream.emit('readable'); - flow(stream); - } - - // at this point, the user has presumably seen the 'readable' event, - // and called read() to consume some data. that may have triggered - // in turn another _read(n) call, in which case reading = true if - // it's in progress. - // However, if we're not ended, or reading, and the length < hwm, - // then go ahead and try to read some more preemptively. - function maybeReadMore(stream, state) { - if (!state.readingMore) { - state.readingMore = true; - nextTick(maybeReadMore_, stream, state); - } - } - - function maybeReadMore_(stream, state) { - var len = state.length; - while (!state.reading && !state.flowing && !state.ended && state.length < state.highWaterMark) { - debug('maybeReadMore read 0'); - stream.read(0); - if (len === state.length) - // didn't get any data, stop spinning. - break;else len = state.length; - } - state.readingMore = false; - } - - // abstract method. to be overridden in specific implementation classes. - // call cb(er, data) where data is <= n in length. - // for virtual (non-string, non-buffer) streams, "length" is somewhat - // arbitrary, and perhaps not very meaningful. - Readable.prototype._read = function (n) { - this.emit('error', new Error('not implemented')); - }; - - Readable.prototype.pipe = function (dest, pipeOpts) { - var src = this; - var state = this._readableState; - - switch (state.pipesCount) { - case 0: - state.pipes = dest; - break; - case 1: - state.pipes = [state.pipes, dest]; - break; - default: - state.pipes.push(dest); - break; - } - state.pipesCount += 1; - debug('pipe count=%d opts=%j', state.pipesCount, pipeOpts); - - var doEnd = (!pipeOpts || pipeOpts.end !== false); - - var endFn = doEnd ? onend : cleanup; - if (state.endEmitted) nextTick(endFn);else src.once('end', endFn); - - dest.on('unpipe', onunpipe); - function onunpipe(readable) { - debug('onunpipe'); - if (readable === src) { - cleanup(); - } - } - - function onend() { - debug('onend'); - dest.end(); - } - - // when the dest drains, it reduces the awaitDrain counter - // on the source. This would be more elegant with a .once() - // handler in flow(), but adding and removing repeatedly is - // too slow. - var ondrain = pipeOnDrain(src); - dest.on('drain', ondrain); - - var cleanedUp = false; - function cleanup() { - debug('cleanup'); - // cleanup event handlers once the pipe is broken - dest.removeListener('close', onclose); - dest.removeListener('finish', onfinish); - dest.removeListener('drain', ondrain); - dest.removeListener('error', onerror); - dest.removeListener('unpipe', onunpipe); - src.removeListener('end', onend); - src.removeListener('end', cleanup); - src.removeListener('data', ondata); - - cleanedUp = true; - - // if the reader is waiting for a drain event from this - // specific writer, then it would cause it to never start - // flowing again. - // So, if this is awaiting a drain, then we just call it now. - // If we don't know, then assume that we are waiting for one. - if (state.awaitDrain && (!dest._writableState || dest._writableState.needDrain)) ondrain(); - } - - // If the user pushes more data while we're writing to dest then we'll end up - // in ondata again. However, we only want to increase awaitDrain once because - // dest will only emit one 'drain' event for the multiple writes. - // => Introduce a guard on increasing awaitDrain. - var increasedAwaitDrain = false; - src.on('data', ondata); - function ondata(chunk) { - debug('ondata'); - increasedAwaitDrain = false; - var ret = dest.write(chunk); - if (false === ret && !increasedAwaitDrain) { - // If the user unpiped during `dest.write()`, it is possible - // to get stuck in a permanently paused state if that write - // also returned false. - // => Check whether `dest` is still a piping destination. - if ((state.pipesCount === 1 && state.pipes === dest || state.pipesCount > 1 && indexOf(state.pipes, dest) !== -1) && !cleanedUp) { - debug('false write response, pause', src._readableState.awaitDrain); - src._readableState.awaitDrain++; - increasedAwaitDrain = true; - } - src.pause(); - } - } - - // if the dest has an error, then stop piping into it. - // however, don't suppress the throwing behavior for this. - function onerror(er) { - debug('onerror', er); - unpipe(); - dest.removeListener('error', onerror); - if (listenerCount(dest, 'error') === 0) dest.emit('error', er); - } - - // Make sure our error handler is attached before userland ones. - prependListener(dest, 'error', onerror); - - // Both close and finish should trigger unpipe, but only once. - function onclose() { - dest.removeListener('finish', onfinish); - unpipe(); - } - dest.once('close', onclose); - function onfinish() { - debug('onfinish'); - dest.removeListener('close', onclose); - unpipe(); - } - dest.once('finish', onfinish); - - function unpipe() { - debug('unpipe'); - src.unpipe(dest); - } - - // tell the dest that it's being piped to - dest.emit('pipe', src); - - // start the flow if it hasn't been started already. - if (!state.flowing) { - debug('pipe resume'); - src.resume(); - } - - return dest; - }; - - function pipeOnDrain(src) { - return function () { - var state = src._readableState; - debug('pipeOnDrain', state.awaitDrain); - if (state.awaitDrain) state.awaitDrain--; - if (state.awaitDrain === 0 && src.listeners('data').length) { - state.flowing = true; - flow(src); - } - }; - } - - Readable.prototype.unpipe = function (dest) { - var state = this._readableState; - - // if we're not piping anywhere, then do nothing. - if (state.pipesCount === 0) return this; - - // just one destination. most common case. - if (state.pipesCount === 1) { - // passed in one, but it's not the right one. - if (dest && dest !== state.pipes) return this; - - if (!dest) dest = state.pipes; - - // got a match. - state.pipes = null; - state.pipesCount = 0; - state.flowing = false; - if (dest) dest.emit('unpipe', this); - return this; - } - - // slow case. multiple pipe destinations. - - if (!dest) { - // remove all. - var dests = state.pipes; - var len = state.pipesCount; - state.pipes = null; - state.pipesCount = 0; - state.flowing = false; - - for (var _i = 0; _i < len; _i++) { - dests[_i].emit('unpipe', this); - }return this; - } - - // try to find the right one. - var i = indexOf(state.pipes, dest); - if (i === -1) return this; - - state.pipes.splice(i, 1); - state.pipesCount -= 1; - if (state.pipesCount === 1) state.pipes = state.pipes[0]; - - dest.emit('unpipe', this); - - return this; - }; - - // set up data events if they are asked for - // Ensure readable listeners eventually get something - Readable.prototype.on = function (ev, fn) { - var res = EventEmitter.prototype.on.call(this, ev, fn); - - if (ev === 'data') { - // Start flowing on next tick if stream isn't explicitly paused - if (this._readableState.flowing !== false) this.resume(); - } else if (ev === 'readable') { - var state = this._readableState; - if (!state.endEmitted && !state.readableListening) { - state.readableListening = state.needReadable = true; - state.emittedReadable = false; - if (!state.reading) { - nextTick(nReadingNextTick, this); - } else if (state.length) { - emitReadable(this); - } - } - } - - return res; - }; - Readable.prototype.addListener = Readable.prototype.on; - - function nReadingNextTick(self) { - debug('readable nexttick read 0'); - self.read(0); - } - - // pause() and resume() are remnants of the legacy readable stream API - // If the user uses them, then switch into old mode. - Readable.prototype.resume = function () { - var state = this._readableState; - if (!state.flowing) { - debug('resume'); - state.flowing = true; - resume(this, state); - } - return this; - }; - - function resume(stream, state) { - if (!state.resumeScheduled) { - state.resumeScheduled = true; - nextTick(resume_, stream, state); - } - } - - function resume_(stream, state) { - if (!state.reading) { - debug('resume read 0'); - stream.read(0); - } - - state.resumeScheduled = false; - state.awaitDrain = 0; - stream.emit('resume'); - flow(stream); - if (state.flowing && !state.reading) stream.read(0); - } - - Readable.prototype.pause = function () { - debug('call pause flowing=%j', this._readableState.flowing); - if (false !== this._readableState.flowing) { - debug('pause'); - this._readableState.flowing = false; - this.emit('pause'); - } - return this; - }; - - function flow(stream) { - var state = stream._readableState; - debug('flow', state.flowing); - while (state.flowing && stream.read() !== null) {} - } - - // wrap an old-style stream as the async data source. - // This is *not* part of the readable stream interface. - // It is an ugly unfortunate mess of history. - Readable.prototype.wrap = function (stream) { - var state = this._readableState; - var paused = false; - - var self = this; - stream.on('end', function () { - debug('wrapped end'); - if (state.decoder && !state.ended) { - var chunk = state.decoder.end(); - if (chunk && chunk.length) self.push(chunk); - } - - self.push(null); - }); - - stream.on('data', function (chunk) { - debug('wrapped data'); - if (state.decoder) chunk = state.decoder.write(chunk); - - // don't skip over falsy values in objectMode - if (state.objectMode && (chunk === null || chunk === undefined)) return;else if (!state.objectMode && (!chunk || !chunk.length)) return; - - var ret = self.push(chunk); - if (!ret) { - paused = true; - stream.pause(); - } - }); - - // proxy all the other methods. - // important when wrapping filters and duplexes. - for (var i in stream) { - if (this[i] === undefined && typeof stream[i] === 'function') { - this[i] = function (method) { - return function () { - return stream[method].apply(stream, arguments); - }; - }(i); - } - } - - // proxy certain important events. - var events = ['error', 'close', 'destroy', 'pause', 'resume']; - forEach(events, function (ev) { - stream.on(ev, self.emit.bind(self, ev)); - }); - - // when we try to consume some more bytes, simply unpause the - // underlying stream. - self._read = function (n) { - debug('wrapped _read', n); - if (paused) { - paused = false; - stream.resume(); - } - }; - - return self; - }; - - // exposed for testing purposes only. - Readable._fromList = fromList; - - // Pluck off n bytes from an array of buffers. - // Length is the combined lengths of all the buffers in the list. - // This function is designed to be inlinable, so please take care when making - // changes to the function body. - function fromList(n, state) { - // nothing buffered - if (state.length === 0) return null; - - var ret; - if (state.objectMode) ret = state.buffer.shift();else if (!n || n >= state.length) { - // read it all, truncate the list - if (state.decoder) ret = state.buffer.join('');else if (state.buffer.length === 1) ret = state.buffer.head.data;else ret = state.buffer.concat(state.length); - state.buffer.clear(); - } else { - // read part of list - ret = fromListPartial(n, state.buffer, state.decoder); - } - - return ret; - } - - // Extracts only enough buffered data to satisfy the amount requested. - // This function is designed to be inlinable, so please take care when making - // changes to the function body. - function fromListPartial(n, list, hasStrings) { - var ret; - if (n < list.head.data.length) { - // slice is the same for buffers and strings - ret = list.head.data.slice(0, n); - list.head.data = list.head.data.slice(n); - } else if (n === list.head.data.length) { - // first chunk is a perfect match - ret = list.shift(); - } else { - // result spans more than one buffer - ret = hasStrings ? copyFromBufferString(n, list) : copyFromBuffer(n, list); - } - return ret; - } - - // Copies a specified amount of characters from the list of buffered data - // chunks. - // This function is designed to be inlinable, so please take care when making - // changes to the function body. - function copyFromBufferString(n, list) { - var p = list.head; - var c = 1; - var ret = p.data; - n -= ret.length; - while (p = p.next) { - var str = p.data; - var nb = n > str.length ? str.length : n; - if (nb === str.length) ret += str;else ret += str.slice(0, n); - n -= nb; - if (n === 0) { - if (nb === str.length) { - ++c; - if (p.next) list.head = p.next;else list.head = list.tail = null; - } else { - list.head = p; - p.data = str.slice(nb); - } - break; - } - ++c; - } - list.length -= c; - return ret; - } - - // Copies a specified amount of bytes from the list of buffered data chunks. - // This function is designed to be inlinable, so please take care when making - // changes to the function body. - function copyFromBuffer(n, list) { - var ret = Buffer.allocUnsafe(n); - var p = list.head; - var c = 1; - p.data.copy(ret); - n -= p.data.length; - while (p = p.next) { - var buf = p.data; - var nb = n > buf.length ? buf.length : n; - buf.copy(ret, ret.length - n, 0, nb); - n -= nb; - if (n === 0) { - if (nb === buf.length) { - ++c; - if (p.next) list.head = p.next;else list.head = list.tail = null; - } else { - list.head = p; - p.data = buf.slice(nb); - } - break; - } - ++c; - } - list.length -= c; - return ret; - } - - function endReadable(stream) { - var state = stream._readableState; - - // If we get here before consuming all the bytes, then that is a - // bug in node. Should never happen. - if (state.length > 0) throw new Error('"endReadable()" called on non-empty stream'); - - if (!state.endEmitted) { - state.ended = true; - nextTick(endReadableNT, state, stream); - } - } - - function endReadableNT(state, stream) { - // Check that we didn't get one last unshift. - if (!state.endEmitted && state.length === 0) { - state.endEmitted = true; - stream.readable = false; - stream.emit('end'); - } - } - - function forEach(xs, f) { - for (var i = 0, l = xs.length; i < l; i++) { - f(xs[i], i); - } - } - - function indexOf(xs, x) { - for (var i = 0, l = xs.length; i < l; i++) { - if (xs[i] === x) return i; - } - return -1; - } - - // A bit simpler than readable streams. - Writable.WritableState = WritableState; - inherits$1(Writable, EventEmitter); - - function nop() {} - - function WriteReq(chunk, encoding, cb) { - this.chunk = chunk; - this.encoding = encoding; - this.callback = cb; - this.next = null; - } - - function WritableState(options, stream) { - Object.defineProperty(this, 'buffer', { - get: deprecate(function () { - return this.getBuffer(); - }, '_writableState.buffer is deprecated. Use _writableState.getBuffer ' + 'instead.') - }); - options = options || {}; - - // object stream flag to indicate whether or not this stream - // contains buffers or objects. - this.objectMode = !!options.objectMode; - - if (stream instanceof Duplex) this.objectMode = this.objectMode || !!options.writableObjectMode; - - // the point at which write() starts returning false - // Note: 0 is a valid value, means that we always return false if - // the entire buffer is not flushed immediately on write() - var hwm = options.highWaterMark; - var defaultHwm = this.objectMode ? 16 : 16 * 1024; - this.highWaterMark = hwm || hwm === 0 ? hwm : defaultHwm; - - // cast to ints. - this.highWaterMark = ~ ~this.highWaterMark; - - this.needDrain = false; - // at the start of calling end() - this.ending = false; - // when end() has been called, and returned - this.ended = false; - // when 'finish' is emitted - this.finished = false; - - // should we decode strings into buffers before passing to _write? - // this is here so that some node-core streams can optimize string - // handling at a lower level. - var noDecode = options.decodeStrings === false; - this.decodeStrings = !noDecode; - - // Crypto is kind of old and crusty. Historically, its default string - // encoding is 'binary' so we have to make this configurable. - // Everything else in the universe uses 'utf8', though. - this.defaultEncoding = options.defaultEncoding || 'utf8'; - - // not an actual buffer we keep track of, but a measurement - // of how much we're waiting to get pushed to some underlying - // socket or file. - this.length = 0; - - // a flag to see when we're in the middle of a write. - this.writing = false; - - // when true all writes will be buffered until .uncork() call - this.corked = 0; - - // a flag to be able to tell if the onwrite cb is called immediately, - // or on a later tick. We set this to true at first, because any - // actions that shouldn't happen until "later" should generally also - // not happen before the first write call. - this.sync = true; - - // a flag to know if we're processing previously buffered items, which - // may call the _write() callback in the same tick, so that we don't - // end up in an overlapped onwrite situation. - this.bufferProcessing = false; - - // the callback that's passed to _write(chunk,cb) - this.onwrite = function (er) { - onwrite(stream, er); - }; - - // the callback that the user supplies to write(chunk,encoding,cb) - this.writecb = null; - - // the amount that is being written when _write is called. - this.writelen = 0; - - this.bufferedRequest = null; - this.lastBufferedRequest = null; - - // number of pending user-supplied write callbacks - // this must be 0 before 'finish' can be emitted - this.pendingcb = 0; - - // emit prefinish if the only thing we're waiting for is _write cbs - // This is relevant for synchronous Transform streams - this.prefinished = false; - - // True if the error was already emitted and should not be thrown again - this.errorEmitted = false; - - // count buffered requests - this.bufferedRequestCount = 0; - - // allocate the first CorkedRequest, there is always - // one allocated and free to use, and we maintain at most two - this.corkedRequestsFree = new CorkedRequest(this); - } - - WritableState.prototype.getBuffer = function writableStateGetBuffer() { - var current = this.bufferedRequest; - var out = []; - while (current) { - out.push(current); - current = current.next; - } - return out; - }; - function Writable(options) { - - // Writable ctor is applied to Duplexes, though they're not - // instanceof Writable, they're instanceof Readable. - if (!(this instanceof Writable) && !(this instanceof Duplex)) return new Writable(options); - - this._writableState = new WritableState(options, this); - - // legacy. - this.writable = true; - - if (options) { - if (typeof options.write === 'function') this._write = options.write; - - if (typeof options.writev === 'function') this._writev = options.writev; - } - - EventEmitter.call(this); - } - - // Otherwise people can pipe Writable streams, which is just wrong. - Writable.prototype.pipe = function () { - this.emit('error', new Error('Cannot pipe, not readable')); - }; - - function writeAfterEnd(stream, cb) { - var er = new Error('write after end'); - // TODO: defer error events consistently everywhere, not just the cb - stream.emit('error', er); - nextTick(cb, er); - } - - // If we get something that is not a buffer, string, null, or undefined, - // and we're not in objectMode, then that's an error. - // Otherwise stream chunks are all considered to be of length=1, and the - // watermarks determine how many objects to keep in the buffer, rather than - // how many bytes or characters. - function validChunk(stream, state, chunk, cb) { - var valid = true; - var er = false; - // Always throw error if a null is written - // if we are not in object mode then throw - // if it is not a buffer, string, or undefined. - if (chunk === null) { - er = new TypeError('May not write null values to stream'); - } else if (!Buffer.isBuffer(chunk) && typeof chunk !== 'string' && chunk !== undefined && !state.objectMode) { - er = new TypeError('Invalid non-string/buffer chunk'); - } - if (er) { - stream.emit('error', er); - nextTick(cb, er); - valid = false; - } - return valid; - } - - Writable.prototype.write = function (chunk, encoding, cb) { - var state = this._writableState; - var ret = false; - - if (typeof encoding === 'function') { - cb = encoding; - encoding = null; - } - - if (Buffer.isBuffer(chunk)) encoding = 'buffer';else if (!encoding) encoding = state.defaultEncoding; - - if (typeof cb !== 'function') cb = nop; - - if (state.ended) writeAfterEnd(this, cb);else if (validChunk(this, state, chunk, cb)) { - state.pendingcb++; - ret = writeOrBuffer(this, state, chunk, encoding, cb); - } - - return ret; - }; - - Writable.prototype.cork = function () { - var state = this._writableState; - - state.corked++; - }; - - Writable.prototype.uncork = function () { - var state = this._writableState; - - if (state.corked) { - state.corked--; - - if (!state.writing && !state.corked && !state.finished && !state.bufferProcessing && state.bufferedRequest) clearBuffer(this, state); - } - }; - - Writable.prototype.setDefaultEncoding = function setDefaultEncoding(encoding) { - // node::ParseEncoding() requires lower case. - if (typeof encoding === 'string') encoding = encoding.toLowerCase(); - if (!(['hex', 'utf8', 'utf-8', 'ascii', 'binary', 'base64', 'ucs2', 'ucs-2', 'utf16le', 'utf-16le', 'raw'].indexOf((encoding + '').toLowerCase()) > -1)) throw new TypeError('Unknown encoding: ' + encoding); - this._writableState.defaultEncoding = encoding; - return this; - }; - - function decodeChunk(state, chunk, encoding) { - if (!state.objectMode && state.decodeStrings !== false && typeof chunk === 'string') { - chunk = Buffer.from(chunk, encoding); - } - return chunk; - } - - // if we're already writing something, then just put this - // in the queue, and wait our turn. Otherwise, call _write - // If we return false, then we need a drain event, so set that flag. - function writeOrBuffer(stream, state, chunk, encoding, cb) { - chunk = decodeChunk(state, chunk, encoding); - - if (Buffer.isBuffer(chunk)) encoding = 'buffer'; - var len = state.objectMode ? 1 : chunk.length; - - state.length += len; - - var ret = state.length < state.highWaterMark; - // we must ensure that previous needDrain will not be reset to false. - if (!ret) state.needDrain = true; - - if (state.writing || state.corked) { - var last = state.lastBufferedRequest; - state.lastBufferedRequest = new WriteReq(chunk, encoding, cb); - if (last) { - last.next = state.lastBufferedRequest; - } else { - state.bufferedRequest = state.lastBufferedRequest; - } - state.bufferedRequestCount += 1; - } else { - doWrite(stream, state, false, len, chunk, encoding, cb); - } - - return ret; - } - - function doWrite(stream, state, writev, len, chunk, encoding, cb) { - state.writelen = len; - state.writecb = cb; - state.writing = true; - state.sync = true; - if (writev) stream._writev(chunk, state.onwrite);else stream._write(chunk, encoding, state.onwrite); - state.sync = false; - } - - function onwriteError(stream, state, sync, er, cb) { - --state.pendingcb; - if (sync) nextTick(cb, er);else cb(er); - - stream._writableState.errorEmitted = true; - stream.emit('error', er); - } - - function onwriteStateUpdate(state) { - state.writing = false; - state.writecb = null; - state.length -= state.writelen; - state.writelen = 0; - } - - function onwrite(stream, er) { - var state = stream._writableState; - var sync = state.sync; - var cb = state.writecb; - - onwriteStateUpdate(state); - - if (er) onwriteError(stream, state, sync, er, cb);else { - // Check if we're actually ready to finish, but don't emit yet - var finished = needFinish(state); - - if (!finished && !state.corked && !state.bufferProcessing && state.bufferedRequest) { - clearBuffer(stream, state); - } - - if (sync) { - /**/ - nextTick(afterWrite, stream, state, finished, cb); - /**/ - } else { - afterWrite(stream, state, finished, cb); - } - } - } - - function afterWrite(stream, state, finished, cb) { - if (!finished) onwriteDrain(stream, state); - state.pendingcb--; - cb(); - finishMaybe(stream, state); - } - - // Must force callback to be called on nextTick, so that we don't - // emit 'drain' before the write() consumer gets the 'false' return - // value, and has a chance to attach a 'drain' listener. - function onwriteDrain(stream, state) { - if (state.length === 0 && state.needDrain) { - state.needDrain = false; - stream.emit('drain'); - } - } - - // if there's something in the buffer waiting, then process it - function clearBuffer(stream, state) { - state.bufferProcessing = true; - var entry = state.bufferedRequest; - - if (stream._writev && entry && entry.next) { - // Fast case, write everything using _writev() - var l = state.bufferedRequestCount; - var buffer = new Array(l); - var holder = state.corkedRequestsFree; - holder.entry = entry; - - var count = 0; - while (entry) { - buffer[count] = entry; - entry = entry.next; - count += 1; - } - - doWrite(stream, state, true, state.length, buffer, '', holder.finish); - - // doWrite is almost always async, defer these to save a bit of time - // as the hot path ends with doWrite - state.pendingcb++; - state.lastBufferedRequest = null; - if (holder.next) { - state.corkedRequestsFree = holder.next; - holder.next = null; - } else { - state.corkedRequestsFree = new CorkedRequest(state); - } - } else { - // Slow case, write chunks one-by-one - while (entry) { - var chunk = entry.chunk; - var encoding = entry.encoding; - var cb = entry.callback; - var len = state.objectMode ? 1 : chunk.length; - - doWrite(stream, state, false, len, chunk, encoding, cb); - entry = entry.next; - // if we didn't call the onwrite immediately, then - // it means that we need to wait until it does. - // also, that means that the chunk and cb are currently - // being processed, so move the buffer counter past them. - if (state.writing) { - break; - } - } - - if (entry === null) state.lastBufferedRequest = null; - } - - state.bufferedRequestCount = 0; - state.bufferedRequest = entry; - state.bufferProcessing = false; - } - - Writable.prototype._write = function (chunk, encoding, cb) { - cb(new Error('not implemented')); - }; - - Writable.prototype._writev = null; - - Writable.prototype.end = function (chunk, encoding, cb) { - var state = this._writableState; - - if (typeof chunk === 'function') { - cb = chunk; - chunk = null; - encoding = null; - } else if (typeof encoding === 'function') { - cb = encoding; - encoding = null; - } - - if (chunk !== null && chunk !== undefined) this.write(chunk, encoding); - - // .end() fully uncorks - if (state.corked) { - state.corked = 1; - this.uncork(); - } - - // ignore unnecessary end() calls. - if (!state.ending && !state.finished) endWritable(this, state, cb); - }; - - function needFinish(state) { - return state.ending && state.length === 0 && state.bufferedRequest === null && !state.finished && !state.writing; - } - - function prefinish(stream, state) { - if (!state.prefinished) { - state.prefinished = true; - stream.emit('prefinish'); - } - } - - function finishMaybe(stream, state) { - var need = needFinish(state); - if (need) { - if (state.pendingcb === 0) { - prefinish(stream, state); - state.finished = true; - stream.emit('finish'); - } else { - prefinish(stream, state); - } - } - return need; - } - - function endWritable(stream, state, cb) { - state.ending = true; - finishMaybe(stream, state); - if (cb) { - if (state.finished) nextTick(cb);else stream.once('finish', cb); - } - state.ended = true; - stream.writable = false; - } - - // It seems a linked list but it is not - // there will be only 2 of these for each stream - function CorkedRequest(state) { - var _this = this; - - this.next = null; - this.entry = null; - - this.finish = function (err) { - var entry = _this.entry; - _this.entry = null; - while (entry) { - var cb = entry.callback; - state.pendingcb--; - cb(err); - entry = entry.next; - } - if (state.corkedRequestsFree) { - state.corkedRequestsFree.next = _this; - } else { - state.corkedRequestsFree = _this; - } - }; - } - - inherits$1(Duplex, Readable); - - var keys = Object.keys(Writable.prototype); - for (var v = 0; v < keys.length; v++) { - var method = keys[v]; - if (!Duplex.prototype[method]) Duplex.prototype[method] = Writable.prototype[method]; - } - function Duplex(options) { - if (!(this instanceof Duplex)) return new Duplex(options); - - Readable.call(this, options); - Writable.call(this, options); - - if (options && options.readable === false) this.readable = false; - - if (options && options.writable === false) this.writable = false; - - this.allowHalfOpen = true; - if (options && options.allowHalfOpen === false) this.allowHalfOpen = false; - - this.once('end', onend); - } - - // the no-half-open enforcer - function onend() { - // if we allow half-open state, or if the writable side ended, - // then we're ok. - if (this.allowHalfOpen || this._writableState.ended) return; - - // no more data can be written. - // But allow more writes to happen in this tick. - nextTick(onEndNT, this); - } - - function onEndNT(self) { - self.end(); - } - - // a transform stream is a readable/writable stream where you do - inherits$1(Transform, Duplex); - - function TransformState(stream) { - this.afterTransform = function (er, data) { - return afterTransform(stream, er, data); - }; - - this.needTransform = false; - this.transforming = false; - this.writecb = null; - this.writechunk = null; - this.writeencoding = null; - } - - function afterTransform(stream, er, data) { - var ts = stream._transformState; - ts.transforming = false; - - var cb = ts.writecb; - - if (!cb) return stream.emit('error', new Error('no writecb in Transform class')); - - ts.writechunk = null; - ts.writecb = null; - - if (data !== null && data !== undefined) stream.push(data); - - cb(er); - - var rs = stream._readableState; - rs.reading = false; - if (rs.needReadable || rs.length < rs.highWaterMark) { - stream._read(rs.highWaterMark); - } - } - function Transform(options) { - if (!(this instanceof Transform)) return new Transform(options); - - Duplex.call(this, options); - - this._transformState = new TransformState(this); - - // when the writable side finishes, then flush out anything remaining. - var stream = this; - - // start out asking for a readable event once data is transformed. - this._readableState.needReadable = true; - - // we have implemented the _read method, and done the other things - // that Readable wants before the first _read call, so unset the - // sync guard flag. - this._readableState.sync = false; - - if (options) { - if (typeof options.transform === 'function') this._transform = options.transform; - - if (typeof options.flush === 'function') this._flush = options.flush; - } - - this.once('prefinish', function () { - if (typeof this._flush === 'function') this._flush(function (er) { - done(stream, er); - });else done(stream); - }); - } - - Transform.prototype.push = function (chunk, encoding) { - this._transformState.needTransform = false; - return Duplex.prototype.push.call(this, chunk, encoding); - }; - - // This is the part where you do stuff! - // override this function in implementation classes. - // 'chunk' is an input chunk. - // - // Call `push(newChunk)` to pass along transformed output - // to the readable side. You may call 'push' zero or more times. - // - // Call `cb(err)` when you are done with this chunk. If you pass - // an error, then that'll put the hurt on the whole operation. If you - // never call cb(), then you'll never get another chunk. - Transform.prototype._transform = function (chunk, encoding, cb) { - throw new Error('Not implemented'); - }; - - Transform.prototype._write = function (chunk, encoding, cb) { - var ts = this._transformState; - ts.writecb = cb; - ts.writechunk = chunk; - ts.writeencoding = encoding; - if (!ts.transforming) { - var rs = this._readableState; - if (ts.needTransform || rs.needReadable || rs.length < rs.highWaterMark) this._read(rs.highWaterMark); - } - }; - - // Doesn't matter what the args are here. - // _transform does all the work. - // That we got here means that the readable side wants more data. - Transform.prototype._read = function (n) { - var ts = this._transformState; - - if (ts.writechunk !== null && ts.writecb && !ts.transforming) { - ts.transforming = true; - this._transform(ts.writechunk, ts.writeencoding, ts.afterTransform); - } else { - // mark that we need a transform, so that any data that comes in - // will get processed, now that we've asked for it. - ts.needTransform = true; + class CsvError extends Error { + constructor(code, message, options, ...contexts) { + if(Array.isArray(message)) message = message.join(' '); + super(message); + if(Error.captureStackTrace !== undefined){ + Error.captureStackTrace(this, CsvError); + } + this.code = code; + for(const context of contexts){ + for(const key in context){ + const value = context[key]; + this[key] = isBuffer(value) ? value.toString(options.encoding) : value == null ? value : JSON.parse(JSON.stringify(value)); + } + } } - }; - - function done(stream, er) { - if (er) return stream.emit('error', er); - - // if there's nothing in the write buffer, then that means - // that nothing more will ever be provided - var ws = stream._writableState; - var ts = stream._transformState; - - if (ws.length) throw new Error('Calling transform done when ws.length != 0'); - - if (ts.transforming) throw new Error('Calling transform done when still transforming'); - - return stream.push(null); } - inherits$1(PassThrough, Transform); - function PassThrough(options) { - if (!(this instanceof PassThrough)) return new PassThrough(options); - - Transform.call(this, options); - } - - PassThrough.prototype._transform = function (chunk, encoding, cb) { - cb(null, chunk); + const is_object = function(obj){ + return (typeof obj === 'object' && obj !== null && !Array.isArray(obj)); }; - inherits$1(Stream, EventEmitter); - Stream.Readable = Readable; - Stream.Writable = Writable; - Stream.Duplex = Duplex; - Stream.Transform = Transform; - Stream.PassThrough = PassThrough; - - // Backwards-compat with node 0.4.x - Stream.Stream = Stream; - - // old-style streams. Note that the pipe method (the only relevant - // part of this class) is overridden in the Readable class. - - function Stream() { - EventEmitter.call(this); - } - - Stream.prototype.pipe = function(dest, options) { - var source = this; - - function ondata(chunk) { - if (dest.writable) { - if (false === dest.write(chunk) && source.pause) { - source.pause(); + const normalize_columns_array = function(columns){ + const normalizedColumns = []; + for(let i = 0, l = columns.length; i < l; i++){ + const column = columns[i]; + if(column === undefined || column === null || column === false){ + normalizedColumns[i] = { disabled: true }; + }else if(typeof column === 'string'){ + normalizedColumns[i] = { name: column }; + }else if(is_object(column)){ + if(typeof column.name !== 'string'){ + throw new CsvError('CSV_OPTION_COLUMNS_MISSING_NAME', [ + 'Option columns missing name:', + `property "name" is required at position ${i}`, + 'when column is an object literal' + ]); } + normalizedColumns[i] = column; + }else { + throw new CsvError('CSV_INVALID_COLUMN_DEFINITION', [ + 'Invalid column definition:', + 'expect a string or a literal object,', + `got ${JSON.stringify(column)} at position ${i}` + ]); } } - - source.on('data', ondata); - - function ondrain() { - if (source.readable && source.resume) { - source.resume(); - } - } - - dest.on('drain', ondrain); - - // If the 'end' option is not supplied, dest.end() will be called when - // source gets the 'end' or 'close' events. Only dest.end() once. - if (!dest._isStdio && (!options || options.end !== false)) { - source.on('end', onend); - source.on('close', onclose); - } - - var didOnEnd = false; - function onend() { - if (didOnEnd) return; - didOnEnd = true; - - dest.end(); - } - - - function onclose() { - if (didOnEnd) return; - didOnEnd = true; - - if (typeof dest.destroy === 'function') dest.destroy(); - } - - // don't leave dangling pipes when there are errors. - function onerror(er) { - cleanup(); - if (EventEmitter.listenerCount(this, 'error') === 0) { - throw er; // Unhandled stream error in pipe. - } - } - - source.on('error', onerror); - dest.on('error', onerror); - - // remove all the event listeners that were added. - function cleanup() { - source.removeListener('data', ondata); - dest.removeListener('drain', ondrain); - - source.removeListener('end', onend); - source.removeListener('close', onclose); - - source.removeListener('error', onerror); - dest.removeListener('error', onerror); - - source.removeListener('end', cleanup); - source.removeListener('close', cleanup); - - dest.removeListener('close', cleanup); - } - - source.on('end', cleanup); - source.on('close', cleanup); - - dest.on('close', cleanup); - - dest.emit('pipe', source); - - // Allow for unix-like usage: A.pipe(B).pipe(C) - return dest; + return normalizedColumns; }; class ResizeableBuffer{ @@ -5031,1232 +2077,1171 @@ var csv_parse_sync = (function (exports) { } } - // white space characters - // https://en.wikipedia.org/wiki/Whitespace_character - // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions/Character_Classes#Types - // \f\n\r\t\v\u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff - const tab = 9; - const nl = 10; // \n, 0x0A in hexadecimal, 10 in decimal - const np = 12; - const cr = 13; // \r, 0x0D in hexadécimal, 13 in decimal - const space = 32; - const boms = { - // Note, the following are equals: - // Buffer.from("\ufeff") - // Buffer.from([239, 187, 191]) - // Buffer.from('EFBBBF', 'hex') - 'utf8': Buffer.from([239, 187, 191]), - // Note, the following are equals: - // Buffer.from "\ufeff", 'utf16le - // Buffer.from([255, 254]) - 'utf16le': Buffer.from([255, 254]) + const init_state = function(options){ + return { + bomSkipped: false, + bufBytesStart: 0, + castField: options.cast_function, + commenting: false, + // Current error encountered by a record + error: undefined, + enabled: options.from_line === 1, + escaping: false, + escapeIsQuote: isBuffer(options.escape) && isBuffer(options.quote) && Buffer.compare(options.escape, options.quote) === 0, + // columns can be `false`, `true`, `Array` + expectedRecordLength: Array.isArray(options.columns) ? options.columns.length : undefined, + field: new ResizeableBuffer(20), + firstLineToHeaders: options.cast_first_line_to_header, + needMoreDataSize: Math.max( + // Skip if the remaining buffer smaller than comment + options.comment !== null ? options.comment.length : 0, + // Skip if the remaining buffer can be delimiter + ...options.delimiter.map((delimiter) => delimiter.length), + // Skip if the remaining buffer can be escape sequence + options.quote !== null ? options.quote.length : 0, + ), + previousBuf: undefined, + quoting: false, + stop: false, + rawBuffer: new ResizeableBuffer(100), + record: [], + recordHasError: false, + record_length: 0, + recordDelimiterMaxLength: options.record_delimiter.length === 0 ? 2 : Math.max(...options.record_delimiter.map((v) => v.length)), + trimChars: [Buffer.from(' ', options.encoding)[0], Buffer.from('\t', options.encoding)[0]], + wasQuoting: false, + wasRowDelimiter: false + }; }; - class CsvError extends Error { - constructor(code, message, options, ...contexts) { - if(Array.isArray(message)) message = message.join(' '); - super(message); - if(Error.captureStackTrace !== undefined){ - Error.captureStackTrace(this, CsvError); - } - this.code = code; - for(const context of contexts){ - for(const key in context){ - const value = context[key]; - this[key] = isBuffer(value) ? value.toString(options.encoding) : value == null ? value : JSON.parse(JSON.stringify(value)); - } - } - } - } - const underscore = function(str){ return str.replace(/([A-Z])/g, function(_, match){ return '_' + match.toLowerCase(); }); }; - const isObject = function(obj){ - return (typeof obj === 'object' && obj !== null && !Array.isArray(obj)); - }; - - const isRecordEmpty = function(record){ - return record.every((field) => field == null || field.toString && field.toString().trim() === ''); - }; - - const normalizeColumnsArray = function(columns){ - const normalizedColumns = []; - for(let i = 0, l = columns.length; i < l; i++){ - const column = columns[i]; - if(column === undefined || column === null || column === false){ - normalizedColumns[i] = { disabled: true }; - }else if(typeof column === 'string'){ - normalizedColumns[i] = { name: column }; - }else if(isObject(column)){ - if(typeof column.name !== 'string'){ - throw new CsvError('CSV_OPTION_COLUMNS_MISSING_NAME', [ - 'Option columns missing name:', - `property "name" is required at position ${i}`, - 'when column is an object literal' - ]); - } - normalizedColumns[i] = column; - }else { - throw new CsvError('CSV_INVALID_COLUMN_DEFINITION', [ - 'Invalid column definition:', - 'expect a string or a literal object,', - `got ${JSON.stringify(column)} at position ${i}` - ]); - } - } - return normalizedColumns; - }; - - class Parser extends Transform { - constructor(opts = {}){ - super({...{readableObjectMode: true}, ...opts, encoding: null}); - this.__originalOptions = opts; - this.__normalizeOptions(opts); - } - __normalizeOptions(opts){ - const options = {}; - // Merge with user options - for(const opt in opts){ - options[underscore(opt)] = opts[opt]; - } - // Normalize option `encoding` - // Note: defined first because other options depends on it - // to convert chars/strings into buffers. - if(options.encoding === undefined || options.encoding === true){ - options.encoding = 'utf8'; - }else if(options.encoding === null || options.encoding === false){ - options.encoding = null; - }else if(typeof options.encoding !== 'string' && options.encoding !== null){ - throw new CsvError('CSV_INVALID_OPTION_ENCODING', [ - 'Invalid option encoding:', - 'encoding must be a string or null to return a buffer,', - `got ${JSON.stringify(options.encoding)}` - ], options); - } - // Normalize option `bom` - if(options.bom === undefined || options.bom === null || options.bom === false){ - options.bom = false; - }else if(options.bom !== true){ - throw new CsvError('CSV_INVALID_OPTION_BOM', [ - 'Invalid option bom:', 'bom must be true,', - `got ${JSON.stringify(options.bom)}` - ], options); - } - // Normalize option `cast` - let fnCastField = null; - if(options.cast === undefined || options.cast === null || options.cast === false || options.cast === ''){ - options.cast = undefined; - }else if(typeof options.cast === 'function'){ - fnCastField = options.cast; - options.cast = true; - }else if(options.cast !== true){ - throw new CsvError('CSV_INVALID_OPTION_CAST', [ - 'Invalid option cast:', 'cast must be true or a function,', - `got ${JSON.stringify(options.cast)}` - ], options); - } - // Normalize option `cast_date` - if(options.cast_date === undefined || options.cast_date === null || options.cast_date === false || options.cast_date === ''){ - options.cast_date = false; - }else if(options.cast_date === true){ - options.cast_date = function(value){ - const date = Date.parse(value); - return !isNaN(date) ? new Date(date) : value; - }; - }else { - throw new CsvError('CSV_INVALID_OPTION_CAST_DATE', [ - 'Invalid option cast_date:', 'cast_date must be true or a function,', - `got ${JSON.stringify(options.cast_date)}` - ], options); - } - // Normalize option `columns` - let fnFirstLineToHeaders = null; - if(options.columns === true){ - // Fields in the first line are converted as-is to columns - fnFirstLineToHeaders = undefined; - }else if(typeof options.columns === 'function'){ - fnFirstLineToHeaders = options.columns; - options.columns = true; - }else if(Array.isArray(options.columns)){ - options.columns = normalizeColumnsArray(options.columns); - }else if(options.columns === undefined || options.columns === null || options.columns === false){ - options.columns = false; - }else { - throw new CsvError('CSV_INVALID_OPTION_COLUMNS', [ - 'Invalid option columns:', - 'expect an array, a function or true,', - `got ${JSON.stringify(options.columns)}` + const normalize_options = function(opts){ + const options = {}; + // Merge with user options + for(const opt in opts){ + options[underscore(opt)] = opts[opt]; + } + // Normalize option `encoding` + // Note: defined first because other options depends on it + // to convert chars/strings into buffers. + if(options.encoding === undefined || options.encoding === true){ + options.encoding = 'utf8'; + }else if(options.encoding === null || options.encoding === false){ + options.encoding = null; + }else if(typeof options.encoding !== 'string' && options.encoding !== null){ + throw new CsvError('CSV_INVALID_OPTION_ENCODING', [ + 'Invalid option encoding:', + 'encoding must be a string or null to return a buffer,', + `got ${JSON.stringify(options.encoding)}` + ], options); + } + // Normalize option `bom` + if(options.bom === undefined || options.bom === null || options.bom === false){ + options.bom = false; + }else if(options.bom !== true){ + throw new CsvError('CSV_INVALID_OPTION_BOM', [ + 'Invalid option bom:', 'bom must be true,', + `got ${JSON.stringify(options.bom)}` + ], options); + } + // Normalize option `cast` + options.cast_function = null; + if(options.cast === undefined || options.cast === null || options.cast === false || options.cast === ''){ + options.cast = undefined; + }else if(typeof options.cast === 'function'){ + options.cast_function = options.cast; + options.cast = true; + }else if(options.cast !== true){ + throw new CsvError('CSV_INVALID_OPTION_CAST', [ + 'Invalid option cast:', 'cast must be true or a function,', + `got ${JSON.stringify(options.cast)}` + ], options); + } + // Normalize option `cast_date` + if(options.cast_date === undefined || options.cast_date === null || options.cast_date === false || options.cast_date === ''){ + options.cast_date = false; + }else if(options.cast_date === true){ + options.cast_date = function(value){ + const date = Date.parse(value); + return !isNaN(date) ? new Date(date) : value; + }; + }else { + throw new CsvError('CSV_INVALID_OPTION_CAST_DATE', [ + 'Invalid option cast_date:', 'cast_date must be true or a function,', + `got ${JSON.stringify(options.cast_date)}` + ], options); + } + // Normalize option `columns` + options.cast_first_line_to_header = null; + if(options.columns === true){ + // Fields in the first line are converted as-is to columns + options.cast_first_line_to_header = undefined; + }else if(typeof options.columns === 'function'){ + options.cast_first_line_to_header = options.columns; + options.columns = true; + }else if(Array.isArray(options.columns)){ + options.columns = normalize_columns_array(options.columns); + }else if(options.columns === undefined || options.columns === null || options.columns === false){ + options.columns = false; + }else { + throw new CsvError('CSV_INVALID_OPTION_COLUMNS', [ + 'Invalid option columns:', + 'expect an array, a function or true,', + `got ${JSON.stringify(options.columns)}` + ], options); + } + // Normalize option `group_columns_by_name` + if(options.group_columns_by_name === undefined || options.group_columns_by_name === null || options.group_columns_by_name === false){ + options.group_columns_by_name = false; + }else if(options.group_columns_by_name !== true){ + throw new CsvError('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ + 'Invalid option group_columns_by_name:', + 'expect an boolean,', + `got ${JSON.stringify(options.group_columns_by_name)}` + ], options); + }else if(options.columns === false){ + throw new CsvError('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ + 'Invalid option group_columns_by_name:', + 'the `columns` mode must be activated.' + ], options); + } + // Normalize option `comment` + if(options.comment === undefined || options.comment === null || options.comment === false || options.comment === ''){ + options.comment = null; + }else { + if(typeof options.comment === 'string'){ + options.comment = Buffer.from(options.comment, options.encoding); + } + if(!isBuffer(options.comment)){ + throw new CsvError('CSV_INVALID_OPTION_COMMENT', [ + 'Invalid option comment:', + 'comment must be a buffer or a string,', + `got ${JSON.stringify(options.comment)}` ], options); } - // Normalize option `group_columns_by_name` - if(options.group_columns_by_name === undefined || options.group_columns_by_name === null || options.group_columns_by_name === false){ - options.group_columns_by_name = false; - }else if(options.group_columns_by_name !== true){ - throw new CsvError('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ - 'Invalid option group_columns_by_name:', - 'expect an boolean,', - `got ${JSON.stringify(options.group_columns_by_name)}` - ], options); - }else if(options.columns === false){ - throw new CsvError('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ - 'Invalid option group_columns_by_name:', - 'the `columns` mode must be activated.' - ], options); + } + // Normalize option `delimiter` + const delimiter_json = JSON.stringify(options.delimiter); + if(!Array.isArray(options.delimiter)) options.delimiter = [options.delimiter]; + if(options.delimiter.length === 0){ + throw new CsvError('CSV_INVALID_OPTION_DELIMITER', [ + 'Invalid option delimiter:', + 'delimiter must be a non empty string or buffer or array of string|buffer,', + `got ${delimiter_json}` + ], options); + } + options.delimiter = options.delimiter.map(function(delimiter){ + if(delimiter === undefined || delimiter === null || delimiter === false){ + return Buffer.from(',', options.encoding); } - // Normalize option `comment` - if(options.comment === undefined || options.comment === null || options.comment === false || options.comment === ''){ - options.comment = null; - }else { - if(typeof options.comment === 'string'){ - options.comment = Buffer.from(options.comment, options.encoding); - } - if(!isBuffer(options.comment)){ - throw new CsvError('CSV_INVALID_OPTION_COMMENT', [ - 'Invalid option comment:', - 'comment must be a buffer or a string,', - `got ${JSON.stringify(options.comment)}` - ], options); - } + if(typeof delimiter === 'string'){ + delimiter = Buffer.from(delimiter, options.encoding); } - // Normalize option `delimiter` - const delimiter_json = JSON.stringify(options.delimiter); - if(!Array.isArray(options.delimiter)) options.delimiter = [options.delimiter]; - if(options.delimiter.length === 0){ + if(!isBuffer(delimiter) || delimiter.length === 0){ throw new CsvError('CSV_INVALID_OPTION_DELIMITER', [ 'Invalid option delimiter:', 'delimiter must be a non empty string or buffer or array of string|buffer,', `got ${delimiter_json}` ], options); } - options.delimiter = options.delimiter.map(function(delimiter){ - if(delimiter === undefined || delimiter === null || delimiter === false){ - return Buffer.from(',', options.encoding); - } - if(typeof delimiter === 'string'){ - delimiter = Buffer.from(delimiter, options.encoding); - } - if(!isBuffer(delimiter) || delimiter.length === 0){ - throw new CsvError('CSV_INVALID_OPTION_DELIMITER', [ - 'Invalid option delimiter:', - 'delimiter must be a non empty string or buffer or array of string|buffer,', - `got ${delimiter_json}` - ], options); - } - return delimiter; - }); - // Normalize option `escape` - if(options.escape === undefined || options.escape === true){ - options.escape = Buffer.from('"', options.encoding); - }else if(typeof options.escape === 'string'){ - options.escape = Buffer.from(options.escape, options.encoding); - }else if (options.escape === null || options.escape === false){ - options.escape = null; - } - if(options.escape !== null){ - if(!isBuffer(options.escape)){ - throw new Error(`Invalid Option: escape must be a buffer, a string or a boolean, got ${JSON.stringify(options.escape)}`); - } - } - // Normalize option `from` - if(options.from === undefined || options.from === null){ - options.from = 1; - }else { - if(typeof options.from === 'string' && /\d+/.test(options.from)){ - options.from = parseInt(options.from); - } - if(Number.isInteger(options.from)){ - if(options.from < 0){ - throw new Error(`Invalid Option: from must be a positive integer, got ${JSON.stringify(opts.from)}`); - } - }else { - throw new Error(`Invalid Option: from must be an integer, got ${JSON.stringify(options.from)}`); - } - } - // Normalize option `from_line` - if(options.from_line === undefined || options.from_line === null){ - options.from_line = 1; - }else { - if(typeof options.from_line === 'string' && /\d+/.test(options.from_line)){ - options.from_line = parseInt(options.from_line); - } - if(Number.isInteger(options.from_line)){ - if(options.from_line <= 0){ - throw new Error(`Invalid Option: from_line must be a positive integer greater than 0, got ${JSON.stringify(opts.from_line)}`); - } - }else { - throw new Error(`Invalid Option: from_line must be an integer, got ${JSON.stringify(opts.from_line)}`); - } - } - // Normalize options `ignore_last_delimiters` - if(options.ignore_last_delimiters === undefined || options.ignore_last_delimiters === null){ - options.ignore_last_delimiters = false; - }else if(typeof options.ignore_last_delimiters === 'number'){ - options.ignore_last_delimiters = Math.floor(options.ignore_last_delimiters); - if(options.ignore_last_delimiters === 0){ - options.ignore_last_delimiters = false; - } - }else if(typeof options.ignore_last_delimiters !== 'boolean'){ - throw new CsvError('CSV_INVALID_OPTION_IGNORE_LAST_DELIMITERS', [ - 'Invalid option `ignore_last_delimiters`:', - 'the value must be a boolean value or an integer,', - `got ${JSON.stringify(options.ignore_last_delimiters)}` - ], options); - } - if(options.ignore_last_delimiters === true && options.columns === false){ - throw new CsvError('CSV_IGNORE_LAST_DELIMITERS_REQUIRES_COLUMNS', [ - 'The option `ignore_last_delimiters`', - 'requires the activation of the `columns` option' - ], options); - } - // Normalize option `info` - if(options.info === undefined || options.info === null || options.info === false){ - options.info = false; - }else if(options.info !== true){ - throw new Error(`Invalid Option: info must be true, got ${JSON.stringify(options.info)}`); - } - // Normalize option `max_record_size` - if(options.max_record_size === undefined || options.max_record_size === null || options.max_record_size === false){ - options.max_record_size = 0; - }else if(Number.isInteger(options.max_record_size) && options.max_record_size >= 0);else if(typeof options.max_record_size === 'string' && /\d+/.test(options.max_record_size)){ - options.max_record_size = parseInt(options.max_record_size); - }else { - throw new Error(`Invalid Option: max_record_size must be a positive integer, got ${JSON.stringify(options.max_record_size)}`); - } - // Normalize option `objname` - if(options.objname === undefined || options.objname === null || options.objname === false){ - options.objname = undefined; - }else if(isBuffer(options.objname)){ - if(options.objname.length === 0){ - throw new Error(`Invalid Option: objname must be a non empty buffer`); - } - if(options.encoding === null);else { - options.objname = options.objname.toString(options.encoding); - } - }else if(typeof options.objname === 'string'){ - if(options.objname.length === 0){ - throw new Error(`Invalid Option: objname must be a non empty string`); - } - // Great, nothing to do - }else if(typeof options.objname === 'number');else { - throw new Error(`Invalid Option: objname must be a string or a buffer, got ${options.objname}`); - } - if(options.objname !== undefined){ - if(typeof options.objname === 'number'){ - if(options.columns !== false){ - throw Error('Invalid Option: objname index cannot be combined with columns or be defined as a field'); - } - }else { // A string or a buffer - if(options.columns === false){ - throw Error('Invalid Option: objname field must be combined with columns or be defined as an index'); - } - } - } - // Normalize option `on_record` - if(options.on_record === undefined || options.on_record === null){ - options.on_record = undefined; - }else if(typeof options.on_record !== 'function'){ - throw new CsvError('CSV_INVALID_OPTION_ON_RECORD', [ - 'Invalid option `on_record`:', - 'expect a function,', - `got ${JSON.stringify(options.on_record)}` - ], options); - } - // Normalize option `quote` - if(options.quote === null || options.quote === false || options.quote === ''){ - options.quote = null; - }else { - if(options.quote === undefined || options.quote === true){ - options.quote = Buffer.from('"', options.encoding); - }else if(typeof options.quote === 'string'){ - options.quote = Buffer.from(options.quote, options.encoding); - } - if(!isBuffer(options.quote)){ - throw new Error(`Invalid Option: quote must be a buffer or a string, got ${JSON.stringify(options.quote)}`); - } - } - // Normalize option `raw` - if(options.raw === undefined || options.raw === null || options.raw === false){ - options.raw = false; - }else if(options.raw !== true){ - throw new Error(`Invalid Option: raw must be true, got ${JSON.stringify(options.raw)}`); - } - // Normalize option `record_delimiter` - if(options.record_delimiter === undefined){ - options.record_delimiter = []; - }else if(typeof options.record_delimiter === 'string' || isBuffer(options.record_delimiter)){ - if(options.record_delimiter.length === 0){ - throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ - 'Invalid option `record_delimiter`:', - 'value must be a non empty string or buffer,', - `got ${JSON.stringify(options.record_delimiter)}` - ], options); - } - options.record_delimiter = [options.record_delimiter]; - }else if(!Array.isArray(options.record_delimiter)){ - throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ - 'Invalid option `record_delimiter`:', - 'value must be a string, a buffer or array of string|buffer,', - `got ${JSON.stringify(options.record_delimiter)}` - ], options); - } - options.record_delimiter = options.record_delimiter.map(function(rd, i){ - if(typeof rd !== 'string' && ! isBuffer(rd)){ - throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ - 'Invalid option `record_delimiter`:', - 'value must be a string, a buffer or array of string|buffer', - `at index ${i},`, - `got ${JSON.stringify(rd)}` - ], options); - }else if(rd.length === 0){ - throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ - 'Invalid option `record_delimiter`:', - 'value must be a non empty string or buffer', - `at index ${i},`, - `got ${JSON.stringify(rd)}` - ], options); - } - if(typeof rd === 'string'){ - rd = Buffer.from(rd, options.encoding); + return delimiter; + }); + // Normalize option `escape` + if(options.escape === undefined || options.escape === true){ + options.escape = Buffer.from('"', options.encoding); + }else if(typeof options.escape === 'string'){ + options.escape = Buffer.from(options.escape, options.encoding); + }else if (options.escape === null || options.escape === false){ + options.escape = null; + } + if(options.escape !== null){ + if(!isBuffer(options.escape)){ + throw new Error(`Invalid Option: escape must be a buffer, a string or a boolean, got ${JSON.stringify(options.escape)}`); + } + } + // Normalize option `from` + if(options.from === undefined || options.from === null){ + options.from = 1; + }else { + if(typeof options.from === 'string' && /\d+/.test(options.from)){ + options.from = parseInt(options.from); + } + if(Number.isInteger(options.from)){ + if(options.from < 0){ + throw new Error(`Invalid Option: from must be a positive integer, got ${JSON.stringify(opts.from)}`); } - return rd; - }); - // Normalize option `relax_column_count` - if(typeof options.relax_column_count === 'boolean');else if(options.relax_column_count === undefined || options.relax_column_count === null){ - options.relax_column_count = false; - }else { - throw new Error(`Invalid Option: relax_column_count must be a boolean, got ${JSON.stringify(options.relax_column_count)}`); - } - if(typeof options.relax_column_count_less === 'boolean');else if(options.relax_column_count_less === undefined || options.relax_column_count_less === null){ - options.relax_column_count_less = false; - }else { - throw new Error(`Invalid Option: relax_column_count_less must be a boolean, got ${JSON.stringify(options.relax_column_count_less)}`); - } - if(typeof options.relax_column_count_more === 'boolean');else if(options.relax_column_count_more === undefined || options.relax_column_count_more === null){ - options.relax_column_count_more = false; - }else { - throw new Error(`Invalid Option: relax_column_count_more must be a boolean, got ${JSON.stringify(options.relax_column_count_more)}`); - } - // Normalize option `relax_quotes` - if(typeof options.relax_quotes === 'boolean');else if(options.relax_quotes === undefined || options.relax_quotes === null){ - options.relax_quotes = false; - }else { - throw new Error(`Invalid Option: relax_quotes must be a boolean, got ${JSON.stringify(options.relax_quotes)}`); - } - // Normalize option `skip_empty_lines` - if(typeof options.skip_empty_lines === 'boolean');else if(options.skip_empty_lines === undefined || options.skip_empty_lines === null){ - options.skip_empty_lines = false; }else { - throw new Error(`Invalid Option: skip_empty_lines must be a boolean, got ${JSON.stringify(options.skip_empty_lines)}`); + throw new Error(`Invalid Option: from must be an integer, got ${JSON.stringify(options.from)}`); } - // Normalize option `skip_records_with_empty_values` - if(typeof options.skip_records_with_empty_values === 'boolean');else if(options.skip_records_with_empty_values === undefined || options.skip_records_with_empty_values === null){ - options.skip_records_with_empty_values = false; - }else { - throw new Error(`Invalid Option: skip_records_with_empty_values must be a boolean, got ${JSON.stringify(options.skip_records_with_empty_values)}`); + } + // Normalize option `from_line` + if(options.from_line === undefined || options.from_line === null){ + options.from_line = 1; + }else { + if(typeof options.from_line === 'string' && /\d+/.test(options.from_line)){ + options.from_line = parseInt(options.from_line); } - // Normalize option `skip_records_with_error` - if(typeof options.skip_records_with_error === 'boolean');else if(options.skip_records_with_error === undefined || options.skip_records_with_error === null){ - options.skip_records_with_error = false; + if(Number.isInteger(options.from_line)){ + if(options.from_line <= 0){ + throw new Error(`Invalid Option: from_line must be a positive integer greater than 0, got ${JSON.stringify(opts.from_line)}`); + } }else { - throw new Error(`Invalid Option: skip_records_with_error must be a boolean, got ${JSON.stringify(options.skip_records_with_error)}`); - } - // Normalize option `rtrim` - if(options.rtrim === undefined || options.rtrim === null || options.rtrim === false){ - options.rtrim = false; - }else if(options.rtrim !== true){ - throw new Error(`Invalid Option: rtrim must be a boolean, got ${JSON.stringify(options.rtrim)}`); + throw new Error(`Invalid Option: from_line must be an integer, got ${JSON.stringify(opts.from_line)}`); } - // Normalize option `ltrim` - if(options.ltrim === undefined || options.ltrim === null || options.ltrim === false){ - options.ltrim = false; - }else if(options.ltrim !== true){ - throw new Error(`Invalid Option: ltrim must be a boolean, got ${JSON.stringify(options.ltrim)}`); + } + // Normalize options `ignore_last_delimiters` + if(options.ignore_last_delimiters === undefined || options.ignore_last_delimiters === null){ + options.ignore_last_delimiters = false; + }else if(typeof options.ignore_last_delimiters === 'number'){ + options.ignore_last_delimiters = Math.floor(options.ignore_last_delimiters); + if(options.ignore_last_delimiters === 0){ + options.ignore_last_delimiters = false; } - // Normalize option `trim` - if(options.trim === undefined || options.trim === null || options.trim === false){ - options.trim = false; - }else if(options.trim !== true){ - throw new Error(`Invalid Option: trim must be a boolean, got ${JSON.stringify(options.trim)}`); + }else if(typeof options.ignore_last_delimiters !== 'boolean'){ + throw new CsvError('CSV_INVALID_OPTION_IGNORE_LAST_DELIMITERS', [ + 'Invalid option `ignore_last_delimiters`:', + 'the value must be a boolean value or an integer,', + `got ${JSON.stringify(options.ignore_last_delimiters)}` + ], options); + } + if(options.ignore_last_delimiters === true && options.columns === false){ + throw new CsvError('CSV_IGNORE_LAST_DELIMITERS_REQUIRES_COLUMNS', [ + 'The option `ignore_last_delimiters`', + 'requires the activation of the `columns` option' + ], options); + } + // Normalize option `info` + if(options.info === undefined || options.info === null || options.info === false){ + options.info = false; + }else if(options.info !== true){ + throw new Error(`Invalid Option: info must be true, got ${JSON.stringify(options.info)}`); + } + // Normalize option `max_record_size` + if(options.max_record_size === undefined || options.max_record_size === null || options.max_record_size === false){ + options.max_record_size = 0; + }else if(Number.isInteger(options.max_record_size) && options.max_record_size >= 0);else if(typeof options.max_record_size === 'string' && /\d+/.test(options.max_record_size)){ + options.max_record_size = parseInt(options.max_record_size); + }else { + throw new Error(`Invalid Option: max_record_size must be a positive integer, got ${JSON.stringify(options.max_record_size)}`); + } + // Normalize option `objname` + if(options.objname === undefined || options.objname === null || options.objname === false){ + options.objname = undefined; + }else if(isBuffer(options.objname)){ + if(options.objname.length === 0){ + throw new Error(`Invalid Option: objname must be a non empty buffer`); + } + if(options.encoding === null);else { + options.objname = options.objname.toString(options.encoding); + } + }else if(typeof options.objname === 'string'){ + if(options.objname.length === 0){ + throw new Error(`Invalid Option: objname must be a non empty string`); + } + // Great, nothing to do + }else if(typeof options.objname === 'number');else { + throw new Error(`Invalid Option: objname must be a string or a buffer, got ${options.objname}`); + } + if(options.objname !== undefined){ + if(typeof options.objname === 'number'){ + if(options.columns !== false){ + throw Error('Invalid Option: objname index cannot be combined with columns or be defined as a field'); + } + }else { // A string or a buffer + if(options.columns === false){ + throw Error('Invalid Option: objname field must be combined with columns or be defined as an index'); + } + } + } + // Normalize option `on_record` + if(options.on_record === undefined || options.on_record === null){ + options.on_record = undefined; + }else if(typeof options.on_record !== 'function'){ + throw new CsvError('CSV_INVALID_OPTION_ON_RECORD', [ + 'Invalid option `on_record`:', + 'expect a function,', + `got ${JSON.stringify(options.on_record)}` + ], options); + } + // Normalize option `quote` + if(options.quote === null || options.quote === false || options.quote === ''){ + options.quote = null; + }else { + if(options.quote === undefined || options.quote === true){ + options.quote = Buffer.from('"', options.encoding); + }else if(typeof options.quote === 'string'){ + options.quote = Buffer.from(options.quote, options.encoding); + } + if(!isBuffer(options.quote)){ + throw new Error(`Invalid Option: quote must be a buffer or a string, got ${JSON.stringify(options.quote)}`); + } + } + // Normalize option `raw` + if(options.raw === undefined || options.raw === null || options.raw === false){ + options.raw = false; + }else if(options.raw !== true){ + throw new Error(`Invalid Option: raw must be true, got ${JSON.stringify(options.raw)}`); + } + // Normalize option `record_delimiter` + if(options.record_delimiter === undefined){ + options.record_delimiter = []; + }else if(typeof options.record_delimiter === 'string' || isBuffer(options.record_delimiter)){ + if(options.record_delimiter.length === 0){ + throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ + 'Invalid option `record_delimiter`:', + 'value must be a non empty string or buffer,', + `got ${JSON.stringify(options.record_delimiter)}` + ], options); } - // Normalize options `trim`, `ltrim` and `rtrim` - if(options.trim === true && opts.ltrim !== false){ - options.ltrim = true; - }else if(options.ltrim !== true){ - options.ltrim = false; + options.record_delimiter = [options.record_delimiter]; + }else if(!Array.isArray(options.record_delimiter)){ + throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ + 'Invalid option `record_delimiter`:', + 'value must be a string, a buffer or array of string|buffer,', + `got ${JSON.stringify(options.record_delimiter)}` + ], options); + } + options.record_delimiter = options.record_delimiter.map(function(rd, i){ + if(typeof rd !== 'string' && ! isBuffer(rd)){ + throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ + 'Invalid option `record_delimiter`:', + 'value must be a string, a buffer or array of string|buffer', + `at index ${i},`, + `got ${JSON.stringify(rd)}` + ], options); + }else if(rd.length === 0){ + throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ + 'Invalid option `record_delimiter`:', + 'value must be a non empty string or buffer', + `at index ${i},`, + `got ${JSON.stringify(rd)}` + ], options); } - if(options.trim === true && opts.rtrim !== false){ - options.rtrim = true; - }else if(options.rtrim !== true){ - options.rtrim = false; + if(typeof rd === 'string'){ + rd = Buffer.from(rd, options.encoding); } - // Normalize option `to` - if(options.to === undefined || options.to === null){ - options.to = -1; - }else { - if(typeof options.to === 'string' && /\d+/.test(options.to)){ - options.to = parseInt(options.to); - } - if(Number.isInteger(options.to)){ - if(options.to <= 0){ - throw new Error(`Invalid Option: to must be a positive integer greater than 0, got ${JSON.stringify(opts.to)}`); - } - }else { - throw new Error(`Invalid Option: to must be an integer, got ${JSON.stringify(opts.to)}`); + return rd; + }); + // Normalize option `relax_column_count` + if(typeof options.relax_column_count === 'boolean');else if(options.relax_column_count === undefined || options.relax_column_count === null){ + options.relax_column_count = false; + }else { + throw new Error(`Invalid Option: relax_column_count must be a boolean, got ${JSON.stringify(options.relax_column_count)}`); + } + if(typeof options.relax_column_count_less === 'boolean');else if(options.relax_column_count_less === undefined || options.relax_column_count_less === null){ + options.relax_column_count_less = false; + }else { + throw new Error(`Invalid Option: relax_column_count_less must be a boolean, got ${JSON.stringify(options.relax_column_count_less)}`); + } + if(typeof options.relax_column_count_more === 'boolean');else if(options.relax_column_count_more === undefined || options.relax_column_count_more === null){ + options.relax_column_count_more = false; + }else { + throw new Error(`Invalid Option: relax_column_count_more must be a boolean, got ${JSON.stringify(options.relax_column_count_more)}`); + } + // Normalize option `relax_quotes` + if(typeof options.relax_quotes === 'boolean');else if(options.relax_quotes === undefined || options.relax_quotes === null){ + options.relax_quotes = false; + }else { + throw new Error(`Invalid Option: relax_quotes must be a boolean, got ${JSON.stringify(options.relax_quotes)}`); + } + // Normalize option `skip_empty_lines` + if(typeof options.skip_empty_lines === 'boolean');else if(options.skip_empty_lines === undefined || options.skip_empty_lines === null){ + options.skip_empty_lines = false; + }else { + throw new Error(`Invalid Option: skip_empty_lines must be a boolean, got ${JSON.stringify(options.skip_empty_lines)}`); + } + // Normalize option `skip_records_with_empty_values` + if(typeof options.skip_records_with_empty_values === 'boolean');else if(options.skip_records_with_empty_values === undefined || options.skip_records_with_empty_values === null){ + options.skip_records_with_empty_values = false; + }else { + throw new Error(`Invalid Option: skip_records_with_empty_values must be a boolean, got ${JSON.stringify(options.skip_records_with_empty_values)}`); + } + // Normalize option `skip_records_with_error` + if(typeof options.skip_records_with_error === 'boolean');else if(options.skip_records_with_error === undefined || options.skip_records_with_error === null){ + options.skip_records_with_error = false; + }else { + throw new Error(`Invalid Option: skip_records_with_error must be a boolean, got ${JSON.stringify(options.skip_records_with_error)}`); + } + // Normalize option `rtrim` + if(options.rtrim === undefined || options.rtrim === null || options.rtrim === false){ + options.rtrim = false; + }else if(options.rtrim !== true){ + throw new Error(`Invalid Option: rtrim must be a boolean, got ${JSON.stringify(options.rtrim)}`); + } + // Normalize option `ltrim` + if(options.ltrim === undefined || options.ltrim === null || options.ltrim === false){ + options.ltrim = false; + }else if(options.ltrim !== true){ + throw new Error(`Invalid Option: ltrim must be a boolean, got ${JSON.stringify(options.ltrim)}`); + } + // Normalize option `trim` + if(options.trim === undefined || options.trim === null || options.trim === false){ + options.trim = false; + }else if(options.trim !== true){ + throw new Error(`Invalid Option: trim must be a boolean, got ${JSON.stringify(options.trim)}`); + } + // Normalize options `trim`, `ltrim` and `rtrim` + if(options.trim === true && opts.ltrim !== false){ + options.ltrim = true; + }else if(options.ltrim !== true){ + options.ltrim = false; + } + if(options.trim === true && opts.rtrim !== false){ + options.rtrim = true; + }else if(options.rtrim !== true){ + options.rtrim = false; + } + // Normalize option `to` + if(options.to === undefined || options.to === null){ + options.to = -1; + }else { + if(typeof options.to === 'string' && /\d+/.test(options.to)){ + options.to = parseInt(options.to); + } + if(Number.isInteger(options.to)){ + if(options.to <= 0){ + throw new Error(`Invalid Option: to must be a positive integer greater than 0, got ${JSON.stringify(opts.to)}`); } - } - // Normalize option `to_line` - if(options.to_line === undefined || options.to_line === null){ - options.to_line = -1; }else { - if(typeof options.to_line === 'string' && /\d+/.test(options.to_line)){ - options.to_line = parseInt(options.to_line); - } - if(Number.isInteger(options.to_line)){ - if(options.to_line <= 0){ - throw new Error(`Invalid Option: to_line must be a positive integer greater than 0, got ${JSON.stringify(opts.to_line)}`); - } - }else { - throw new Error(`Invalid Option: to_line must be an integer, got ${JSON.stringify(opts.to_line)}`); - } - } - this.info = { - bytes: 0, - comment_lines: 0, - empty_lines: 0, - invalid_field_length: 0, - lines: 1, - records: 0 - }; - this.options = options; - this.state = { - bomSkipped: false, - bufBytesStart: 0, - castField: fnCastField, - commenting: false, - // Current error encountered by a record - error: undefined, - enabled: options.from_line === 1, - escaping: false, - escapeIsQuote: isBuffer(options.escape) && isBuffer(options.quote) && Buffer.compare(options.escape, options.quote) === 0, - // columns can be `false`, `true`, `Array` - expectedRecordLength: Array.isArray(options.columns) ? options.columns.length : undefined, - field: new ResizeableBuffer(20), - firstLineToHeaders: fnFirstLineToHeaders, - needMoreDataSize: Math.max( - // Skip if the remaining buffer smaller than comment - options.comment !== null ? options.comment.length : 0, - // Skip if the remaining buffer can be delimiter - ...options.delimiter.map((delimiter) => delimiter.length), - // Skip if the remaining buffer can be escape sequence - options.quote !== null ? options.quote.length : 0, - ), - previousBuf: undefined, - quoting: false, - stop: false, - rawBuffer: new ResizeableBuffer(100), - record: [], - recordHasError: false, - record_length: 0, - recordDelimiterMaxLength: options.record_delimiter.length === 0 ? 2 : Math.max(...options.record_delimiter.map((v) => v.length)), - trimChars: [Buffer.from(' ', options.encoding)[0], Buffer.from('\t', options.encoding)[0]], - wasQuoting: false, - wasRowDelimiter: false - }; - } - // Implementation of `Transform._transform` - _transform(buf, encoding, callback){ - if(this.state.stop === true){ - return; - } - const err = this.__parse(buf, false); - if(err !== undefined){ - this.state.stop = true; + throw new Error(`Invalid Option: to must be an integer, got ${JSON.stringify(opts.to)}`); } - callback(err); } - // Implementation of `Transform._flush` - _flush(callback){ - if(this.state.stop === true){ - return; + // Normalize option `to_line` + if(options.to_line === undefined || options.to_line === null){ + options.to_line = -1; + }else { + if(typeof options.to_line === 'string' && /\d+/.test(options.to_line)){ + options.to_line = parseInt(options.to_line); } - const err = this.__parse(undefined, true); - callback(err); - } - // Central parser implementation - __parse(nextBuf, end){ - const {bom, comment, escape, from_line, ltrim, max_record_size, quote, raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options; - let {record_delimiter} = this.options; - const {bomSkipped, previousBuf, rawBuffer, escapeIsQuote} = this.state; - let buf; - if(previousBuf === undefined){ - if(nextBuf === undefined){ - // Handle empty string - this.push(null); - return; - }else { - buf = nextBuf; + if(Number.isInteger(options.to_line)){ + if(options.to_line <= 0){ + throw new Error(`Invalid Option: to_line must be a positive integer greater than 0, got ${JSON.stringify(opts.to_line)}`); } - }else if(previousBuf !== undefined && nextBuf === undefined){ - buf = previousBuf; }else { - buf = Buffer.concat([previousBuf, nextBuf]); + throw new Error(`Invalid Option: to_line must be an integer, got ${JSON.stringify(opts.to_line)}`); } - // Handle UTF BOM - if(bomSkipped === false){ - if(bom === false){ - this.state.bomSkipped = true; - }else if(buf.length < 3){ - // No enough data - if(end === false){ - // Wait for more data - this.state.previousBuf = buf; + } + return options; + }; + + const isRecordEmpty = function(record){ + return record.every((field) => field == null || field.toString && field.toString().trim() === ''); + }; + + // white space characters + // https://en.wikipedia.org/wiki/Whitespace_character + // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions/Character_Classes#Types + // \f\n\r\t\v\u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff + const tab = 9; + const nl = 10; // \n, 0x0A in hexadecimal, 10 in decimal + const np = 12; + const cr = 13; // \r, 0x0D in hexadécimal, 13 in decimal + const space = 32; + const boms = { + // Note, the following are equals: + // Buffer.from("\ufeff") + // Buffer.from([239, 187, 191]) + // Buffer.from('EFBBBF', 'hex') + 'utf8': Buffer.from([239, 187, 191]), + // Note, the following are equals: + // Buffer.from "\ufeff", 'utf16le + // Buffer.from([255, 254]) + 'utf16le': Buffer.from([255, 254]) + }; + + const transform = function(original_options = {}) { + const info = { + bytes: 0, + comment_lines: 0, + empty_lines: 0, + invalid_field_length: 0, + lines: 1, + records: 0 + }; + const options = normalize_options(original_options); + return { + info: info, + original_options: original_options, + options: options, + state: init_state(options), + __needMoreData: function(i, bufLen, end){ + if(end) return false; + const {quote} = this.options; + const {quoting, needMoreDataSize, recordDelimiterMaxLength} = this.state; + const numOfCharLeft = bufLen - i - 1; + const requiredLength = Math.max( + needMoreDataSize, + // Skip if the remaining buffer smaller than record delimiter + recordDelimiterMaxLength, + // Skip if the remaining buffer can be record delimiter following the closing quote + // 1 is for quote.length + quoting ? (quote.length + recordDelimiterMaxLength) : 0, + ); + return numOfCharLeft < requiredLength; + }, + // Central parser implementation + parse: function(nextBuf, end, push, close){ + const {bom, comment, escape, from_line, ltrim, max_record_size, quote, raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options; + let {record_delimiter} = this.options; + const {bomSkipped, previousBuf, rawBuffer, escapeIsQuote} = this.state; + let buf; + if(previousBuf === undefined){ + if(nextBuf === undefined){ + // Handle empty string + close(); return; + }else { + buf = nextBuf; } + }else if(previousBuf !== undefined && nextBuf === undefined){ + buf = previousBuf; }else { - for(const encoding in boms){ - if(boms[encoding].compare(buf, 0, boms[encoding].length) === 0){ - // Skip BOM - const bomLength = boms[encoding].length; - this.state.bufBytesStart += bomLength; - buf = buf.slice(bomLength); - // Renormalize original options with the new encoding - this.__normalizeOptions({...this.__originalOptions, encoding: encoding}); - break; + buf = Buffer.concat([previousBuf, nextBuf]); + } + // Handle UTF BOM + if(bomSkipped === false){ + if(bom === false){ + this.state.bomSkipped = true; + }else if(buf.length < 3){ + // No enough data + if(end === false){ + // Wait for more data + this.state.previousBuf = buf; + return; + } + }else { + for(const encoding in boms){ + if(boms[encoding].compare(buf, 0, boms[encoding].length) === 0){ + // Skip BOM + const bomLength = boms[encoding].length; + this.state.bufBytesStart += bomLength; + buf = buf.slice(bomLength); + // Renormalize original options with the new encoding + this.options = normalize_options({...this.original_options, encoding: encoding}); + break; + } } + this.state.bomSkipped = true; } - this.state.bomSkipped = true; - } - } - const bufLen = buf.length; - let pos; - for(pos = 0; pos < bufLen; pos++){ - // Ensure we get enough space to look ahead - // There should be a way to move this out of the loop - if(this.__needMoreData(pos, bufLen, end)){ - break; - } - if(this.state.wasRowDelimiter === true){ - this.info.lines++; - this.state.wasRowDelimiter = false; - } - if(to_line !== -1 && this.info.lines > to_line){ - this.state.stop = true; - this.push(null); - return; } - // Auto discovery of record_delimiter, unix, mac and windows supported - if(this.state.quoting === false && record_delimiter.length === 0){ - const record_delimiterCount = this.__autoDiscoverRecordDelimiter(buf, pos); - if(record_delimiterCount){ - record_delimiter = this.options.record_delimiter; + const bufLen = buf.length; + let pos; + for(pos = 0; pos < bufLen; pos++){ + // Ensure we get enough space to look ahead + // There should be a way to move this out of the loop + if(this.__needMoreData(pos, bufLen, end)){ + break; } - } - const chr = buf[pos]; - if(raw === true){ - rawBuffer.append(chr); - } - if((chr === cr || chr === nl) && this.state.wasRowDelimiter === false){ - this.state.wasRowDelimiter = true; - } - // Previous char was a valid escape char - // treat the current char as a regular char - if(this.state.escaping === true){ - this.state.escaping = false; - }else { - // Escape is only active inside quoted fields - // We are quoting, the char is an escape chr and there is a chr to escape - // if(escape !== null && this.state.quoting === true && chr === escape && pos + 1 < bufLen){ - if(escape !== null && this.state.quoting === true && this.__isEscape(buf, pos, chr) && pos + escape.length < bufLen){ - if(escapeIsQuote){ - if(this.__isQuote(buf, pos+escape.length)){ - this.state.escaping = true; - pos += escape.length - 1; - continue; - } - }else { - this.state.escaping = true; - pos += escape.length - 1; - continue; + if(this.state.wasRowDelimiter === true){ + this.info.lines++; + this.state.wasRowDelimiter = false; + } + if(to_line !== -1 && this.info.lines > to_line){ + this.state.stop = true; + close(); + return; + } + // Auto discovery of record_delimiter, unix, mac and windows supported + if(this.state.quoting === false && record_delimiter.length === 0){ + const record_delimiterCount = this.__autoDiscoverRecordDelimiter(buf, pos); + if(record_delimiterCount){ + record_delimiter = this.options.record_delimiter; } } - // Not currently escaping and chr is a quote - // TODO: need to compare bytes instead of single char - if(this.state.commenting === false && this.__isQuote(buf, pos)){ - if(this.state.quoting === true){ - const nextChr = buf[pos+quote.length]; - const isNextChrTrimable = rtrim && this.__isCharTrimable(nextChr); - const isNextChrComment = comment !== null && this.__compareBytes(comment, buf, pos+quote.length, nextChr); - const isNextChrDelimiter = this.__isDelimiter(buf, pos+quote.length, nextChr); - const isNextChrRecordDelimiter = record_delimiter.length === 0 ? this.__autoDiscoverRecordDelimiter(buf, pos+quote.length) : this.__isRecordDelimiter(nextChr, buf, pos+quote.length); - // Escape a quote - // Treat next char as a regular character - if(escape !== null && this.__isEscape(buf, pos, chr) && this.__isQuote(buf, pos + escape.length)){ + const chr = buf[pos]; + if(raw === true){ + rawBuffer.append(chr); + } + if((chr === cr || chr === nl) && this.state.wasRowDelimiter === false){ + this.state.wasRowDelimiter = true; + } + // Previous char was a valid escape char + // treat the current char as a regular char + if(this.state.escaping === true){ + this.state.escaping = false; + }else { + // Escape is only active inside quoted fields + // We are quoting, the char is an escape chr and there is a chr to escape + // if(escape !== null && this.state.quoting === true && chr === escape && pos + 1 < bufLen){ + if(escape !== null && this.state.quoting === true && this.__isEscape(buf, pos, chr) && pos + escape.length < bufLen){ + if(escapeIsQuote){ + if(this.__isQuote(buf, pos+escape.length)){ + this.state.escaping = true; + pos += escape.length - 1; + continue; + } + }else { + this.state.escaping = true; pos += escape.length - 1; - }else if(!nextChr || isNextChrDelimiter || isNextChrRecordDelimiter || isNextChrComment || isNextChrTrimable){ - this.state.quoting = false; - this.state.wasQuoting = true; - pos += quote.length - 1; continue; - }else if(relax_quotes === false){ - const err = this.__error( - new CsvError('CSV_INVALID_CLOSING_QUOTE', [ - 'Invalid Closing Quote:', - `got "${String.fromCharCode(nextChr)}"`, - `at line ${this.info.lines}`, - 'instead of delimiter, record delimiter, trimable character', - '(if activated) or comment', - ], this.options, this.__infoField()) - ); - if(err !== undefined) return err; - }else { - this.state.quoting = false; - this.state.wasQuoting = true; - this.state.field.prepend(quote); - pos += quote.length - 1; } - }else { - if(this.state.field.length !== 0){ - // In relax_quotes mode, treat opening quote preceded by chrs as regular - if(relax_quotes === false){ + } + // Not currently escaping and chr is a quote + // TODO: need to compare bytes instead of single char + if(this.state.commenting === false && this.__isQuote(buf, pos)){ + if(this.state.quoting === true){ + const nextChr = buf[pos+quote.length]; + const isNextChrTrimable = rtrim && this.__isCharTrimable(nextChr); + const isNextChrComment = comment !== null && this.__compareBytes(comment, buf, pos+quote.length, nextChr); + const isNextChrDelimiter = this.__isDelimiter(buf, pos+quote.length, nextChr); + const isNextChrRecordDelimiter = record_delimiter.length === 0 ? this.__autoDiscoverRecordDelimiter(buf, pos+quote.length) : this.__isRecordDelimiter(nextChr, buf, pos+quote.length); + // Escape a quote + // Treat next char as a regular character + if(escape !== null && this.__isEscape(buf, pos, chr) && this.__isQuote(buf, pos + escape.length)){ + pos += escape.length - 1; + }else if(!nextChr || isNextChrDelimiter || isNextChrRecordDelimiter || isNextChrComment || isNextChrTrimable){ + this.state.quoting = false; + this.state.wasQuoting = true; + pos += quote.length - 1; + continue; + }else if(relax_quotes === false){ const err = this.__error( - new CsvError('INVALID_OPENING_QUOTE', [ - 'Invalid Opening Quote:', - `a quote is found inside a field at line ${this.info.lines}`, - ], this.options, this.__infoField(), { - field: this.state.field, - }) + new CsvError('CSV_INVALID_CLOSING_QUOTE', [ + 'Invalid Closing Quote:', + `got "${String.fromCharCode(nextChr)}"`, + `at line ${this.info.lines}`, + 'instead of delimiter, record delimiter, trimable character', + '(if activated) or comment', + ], this.options, this.__infoField()) ); if(err !== undefined) return err; + }else { + this.state.quoting = false; + this.state.wasQuoting = true; + this.state.field.prepend(quote); + pos += quote.length - 1; } }else { - this.state.quoting = true; - pos += quote.length - 1; - continue; - } - } - } - if(this.state.quoting === false){ - const recordDelimiterLength = this.__isRecordDelimiter(chr, buf, pos); - if(recordDelimiterLength !== 0){ - // Do not emit comments which take a full line - const skipCommentLine = this.state.commenting && (this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0); - if(skipCommentLine){ - this.info.comment_lines++; - // Skip full comment line - }else { - // Activate records emition if above from_line - if(this.state.enabled === false && this.info.lines + (this.state.wasRowDelimiter === true ? 1: 0) >= from_line){ - this.state.enabled = true; - this.__resetField(); - this.__resetRecord(); - pos += recordDelimiterLength - 1; + if(this.state.field.length !== 0){ + // In relax_quotes mode, treat opening quote preceded by chrs as regular + if(relax_quotes === false){ + const err = this.__error( + new CsvError('INVALID_OPENING_QUOTE', [ + 'Invalid Opening Quote:', + `a quote is found inside a field at line ${this.info.lines}`, + ], this.options, this.__infoField(), { + field: this.state.field, + }) + ); + if(err !== undefined) return err; + } + }else { + this.state.quoting = true; + pos += quote.length - 1; continue; } - // Skip if line is empty and skip_empty_lines activated - if(skip_empty_lines === true && this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0){ - this.info.empty_lines++; - pos += recordDelimiterLength - 1; - continue; + } + } + if(this.state.quoting === false){ + const recordDelimiterLength = this.__isRecordDelimiter(chr, buf, pos); + if(recordDelimiterLength !== 0){ + // Do not emit comments which take a full line + const skipCommentLine = this.state.commenting && (this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0); + if(skipCommentLine){ + this.info.comment_lines++; + // Skip full comment line + }else { + // Activate records emition if above from_line + if(this.state.enabled === false && this.info.lines + (this.state.wasRowDelimiter === true ? 1: 0) >= from_line){ + this.state.enabled = true; + this.__resetField(); + this.__resetRecord(); + pos += recordDelimiterLength - 1; + continue; + } + // Skip if line is empty and skip_empty_lines activated + if(skip_empty_lines === true && this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0){ + this.info.empty_lines++; + pos += recordDelimiterLength - 1; + continue; + } + this.info.bytes = this.state.bufBytesStart + pos; + const errField = this.__onField(); + if(errField !== undefined) return errField; + this.info.bytes = this.state.bufBytesStart + pos + recordDelimiterLength; + const errRecord = this.__onRecord(push); + if(errRecord !== undefined) return errRecord; + if(to !== -1 && this.info.records >= to){ + this.state.stop = true; + close(); + return; + } } + this.state.commenting = false; + pos += recordDelimiterLength - 1; + continue; + } + if(this.state.commenting){ + continue; + } + const commentCount = comment === null ? 0 : this.__compareBytes(comment, buf, pos, chr); + if(commentCount !== 0){ + this.state.commenting = true; + continue; + } + const delimiterLength = this.__isDelimiter(buf, pos, chr); + if(delimiterLength !== 0){ this.info.bytes = this.state.bufBytesStart + pos; const errField = this.__onField(); if(errField !== undefined) return errField; - this.info.bytes = this.state.bufBytesStart + pos + recordDelimiterLength; - const errRecord = this.__onRecord(); - if(errRecord !== undefined) return errRecord; - if(to !== -1 && this.info.records >= to){ - this.state.stop = true; - this.push(null); - return; - } + pos += delimiterLength - 1; + continue; } - this.state.commenting = false; - pos += recordDelimiterLength - 1; - continue; - } - if(this.state.commenting){ - continue; } - const commentCount = comment === null ? 0 : this.__compareBytes(comment, buf, pos, chr); - if(commentCount !== 0){ - this.state.commenting = true; - continue; - } - const delimiterLength = this.__isDelimiter(buf, pos, chr); - if(delimiterLength !== 0){ - this.info.bytes = this.state.bufBytesStart + pos; - const errField = this.__onField(); - if(errField !== undefined) return errField; - pos += delimiterLength - 1; - continue; + } + if(this.state.commenting === false){ + if(max_record_size !== 0 && this.state.record_length + this.state.field.length > max_record_size){ + const err = this.__error( + new CsvError('CSV_MAX_RECORD_SIZE', [ + 'Max Record Size:', + 'record exceed the maximum number of tolerated bytes', + `of ${max_record_size}`, + `at line ${this.info.lines}`, + ], this.options, this.__infoField()) + ); + if(err !== undefined) return err; } } - } - if(this.state.commenting === false){ - if(max_record_size !== 0 && this.state.record_length + this.state.field.length > max_record_size){ + const lappend = ltrim === false || this.state.quoting === true || this.state.field.length !== 0 || !this.__isCharTrimable(chr); + // rtrim in non quoting is handle in __onField + const rappend = rtrim === false || this.state.wasQuoting === false; + if(lappend === true && rappend === true){ + this.state.field.append(chr); + }else if(rtrim === true && !this.__isCharTrimable(chr)){ const err = this.__error( - new CsvError('CSV_MAX_RECORD_SIZE', [ - 'Max Record Size:', - 'record exceed the maximum number of tolerated bytes', - `of ${max_record_size}`, + new CsvError('CSV_NON_TRIMABLE_CHAR_AFTER_CLOSING_QUOTE', [ + 'Invalid Closing Quote:', + 'found non trimable byte after quote', `at line ${this.info.lines}`, ], this.options, this.__infoField()) ); if(err !== undefined) return err; } } - const lappend = ltrim === false || this.state.quoting === true || this.state.field.length !== 0 || !this.__isCharTrimable(chr); - // rtrim in non quoting is handle in __onField - const rappend = rtrim === false || this.state.wasQuoting === false; - if(lappend === true && rappend === true){ - this.state.field.append(chr); - }else if(rtrim === true && !this.__isCharTrimable(chr)){ - const err = this.__error( - new CsvError('CSV_NON_TRIMABLE_CHAR_AFTER_CLOSING_QUOTE', [ - 'Invalid Closing Quote:', - 'found non trimable byte after quote', - `at line ${this.info.lines}`, - ], this.options, this.__infoField()) - ); - if(err !== undefined) return err; - } - } - if(end === true){ - // Ensure we are not ending in a quoting state - if(this.state.quoting === true){ - const err = this.__error( - new CsvError('CSV_QUOTE_NOT_CLOSED', [ - 'Quote Not Closed:', - `the parsing is finished with an opening quote at line ${this.info.lines}`, - ], this.options, this.__infoField()) - ); - if(err !== undefined) return err; + if(end === true){ + // Ensure we are not ending in a quoting state + if(this.state.quoting === true){ + const err = this.__error( + new CsvError('CSV_QUOTE_NOT_CLOSED', [ + 'Quote Not Closed:', + `the parsing is finished with an opening quote at line ${this.info.lines}`, + ], this.options, this.__infoField()) + ); + if(err !== undefined) return err; + }else { + // Skip last line if it has no characters + if(this.state.wasQuoting === true || this.state.record.length !== 0 || this.state.field.length !== 0){ + this.info.bytes = this.state.bufBytesStart + pos; + const errField = this.__onField(); + if(errField !== undefined) return errField; + const errRecord = this.__onRecord(push); + if(errRecord !== undefined) return errRecord; + }else if(this.state.wasRowDelimiter === true){ + this.info.empty_lines++; + }else if(this.state.commenting === true){ + this.info.comment_lines++; + } + } }else { - // Skip last line if it has no characters - if(this.state.wasQuoting === true || this.state.record.length !== 0 || this.state.field.length !== 0){ - this.info.bytes = this.state.bufBytesStart + pos; - const errField = this.__onField(); - if(errField !== undefined) return errField; - const errRecord = this.__onRecord(); - if(errRecord !== undefined) return errRecord; - }else if(this.state.wasRowDelimiter === true){ - this.info.empty_lines++; - }else if(this.state.commenting === true){ - this.info.comment_lines++; + this.state.bufBytesStart += pos; + this.state.previousBuf = buf.slice(pos); + } + if(this.state.wasRowDelimiter === true){ + this.info.lines++; + this.state.wasRowDelimiter = false; + } + }, + __onRecord: function(push){ + const {columns, group_columns_by_name, encoding, info, from, relax_column_count, relax_column_count_less, relax_column_count_more, raw, skip_records_with_empty_values} = this.options; + const {enabled, record} = this.state; + if(enabled === false){ + return this.__resetRecord(); + } + // Convert the first line into column names + const recordLength = record.length; + if(columns === true){ + if(skip_records_with_empty_values === true && isRecordEmpty(record)){ + this.__resetRecord(); + return; + } + return this.__firstLineToColumns(record); + } + if(columns === false && this.info.records === 0){ + this.state.expectedRecordLength = recordLength; + } + if(recordLength !== this.state.expectedRecordLength){ + const err = columns === false ? + new CsvError('CSV_RECORD_INCONSISTENT_FIELDS_LENGTH', [ + 'Invalid Record Length:', + `expect ${this.state.expectedRecordLength},`, + `got ${recordLength} on line ${this.info.lines}`, + ], this.options, this.__infoField(), { + record: record, + }) + : + new CsvError('CSV_RECORD_INCONSISTENT_COLUMNS', [ + 'Invalid Record Length:', + `columns length is ${columns.length},`, // rename columns + `got ${recordLength} on line ${this.info.lines}`, + ], this.options, this.__infoField(), { + record: record, + }); + if(relax_column_count === true || + (relax_column_count_less === true && recordLength < this.state.expectedRecordLength) || + (relax_column_count_more === true && recordLength > this.state.expectedRecordLength)){ + this.info.invalid_field_length++; + this.state.error = err; + // Error is undefined with skip_records_with_error + }else { + const finalErr = this.__error(err); + if(finalErr) return finalErr; } } - }else { - this.state.bufBytesStart += pos; - this.state.previousBuf = buf.slice(pos); - } - if(this.state.wasRowDelimiter === true){ - this.info.lines++; - this.state.wasRowDelimiter = false; - } - } - __onRecord(){ - const {columns, group_columns_by_name, encoding, info, from, relax_column_count, relax_column_count_less, relax_column_count_more, raw, skip_records_with_empty_values} = this.options; - const {enabled, record} = this.state; - if(enabled === false){ - return this.__resetRecord(); - } - // Convert the first line into column names - const recordLength = record.length; - if(columns === true){ if(skip_records_with_empty_values === true && isRecordEmpty(record)){ this.__resetRecord(); return; } - return this.__firstLineToColumns(record); - } - if(columns === false && this.info.records === 0){ - this.state.expectedRecordLength = recordLength; - } - if(recordLength !== this.state.expectedRecordLength){ - const err = columns === false ? - new CsvError('CSV_RECORD_INCONSISTENT_FIELDS_LENGTH', [ - 'Invalid Record Length:', - `expect ${this.state.expectedRecordLength},`, - `got ${recordLength} on line ${this.info.lines}`, - ], this.options, this.__infoField(), { - record: record, - }) - : - new CsvError('CSV_RECORD_INCONSISTENT_COLUMNS', [ - 'Invalid Record Length:', - `columns length is ${columns.length},`, // rename columns - `got ${recordLength} on line ${this.info.lines}`, - ], this.options, this.__infoField(), { - record: record, - }); - if(relax_column_count === true || - (relax_column_count_less === true && recordLength < this.state.expectedRecordLength) || - (relax_column_count_more === true && recordLength > this.state.expectedRecordLength)){ - this.info.invalid_field_length++; - this.state.error = err; - // Error is undefined with skip_records_with_error - }else { - const finalErr = this.__error(err); - if(finalErr) return finalErr; + if(this.state.recordHasError === true){ + this.__resetRecord(); + this.state.recordHasError = false; + return; } - } - if(skip_records_with_empty_values === true && isRecordEmpty(record)){ - this.__resetRecord(); - return; - } - if(this.state.recordHasError === true){ - this.__resetRecord(); - this.state.recordHasError = false; - return; - } - this.info.records++; - if(from === 1 || this.info.records >= from){ - const {objname} = this.options; - // With columns, records are object - if(columns !== false){ - const obj = {}; - // Transform record array to an object - for(let i = 0, l = record.length; i < l; i++){ - if(columns[i] === undefined || columns[i].disabled) continue; - // Turn duplicate columns into an array - if (group_columns_by_name === true && obj[columns[i].name] !== undefined) { - if (Array.isArray(obj[columns[i].name])) { - obj[columns[i].name] = obj[columns[i].name].concat(record[i]); + this.info.records++; + if(from === 1 || this.info.records >= from){ + const {objname} = this.options; + // With columns, records are object + if(columns !== false){ + const obj = {}; + // Transform record array to an object + for(let i = 0, l = record.length; i < l; i++){ + if(columns[i] === undefined || columns[i].disabled) continue; + // Turn duplicate columns into an array + if (group_columns_by_name === true && obj[columns[i].name] !== undefined) { + if (Array.isArray(obj[columns[i].name])) { + obj[columns[i].name] = obj[columns[i].name].concat(record[i]); + } else { + obj[columns[i].name] = [obj[columns[i].name], record[i]]; + } } else { - obj[columns[i].name] = [obj[columns[i].name], record[i]]; + obj[columns[i].name] = record[i]; } - } else { - obj[columns[i].name] = record[i]; - } - } - // Without objname (default) - if(raw === true || info === true){ - const extRecord = Object.assign( - {record: obj}, - (raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}), - (info === true ? {info: this.__infoRecord()}: {}) - ); - const err = this.__push( - objname === undefined ? extRecord : [obj[objname], extRecord] - ); - if(err){ - return err; - } - }else { - const err = this.__push( - objname === undefined ? obj : [obj[objname], obj] - ); - if(err){ - return err; } - } - // Without columns, records are array - }else { - if(raw === true || info === true){ - const extRecord = Object.assign( - {record: record}, - raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}, - info === true ? {info: this.__infoRecord()}: {} - ); - const err = this.__push( - objname === undefined ? extRecord : [record[objname], extRecord] - ); - if(err){ - return err; + // Without objname (default) + if(raw === true || info === true){ + const extRecord = Object.assign( + {record: obj}, + (raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}), + (info === true ? {info: this.__infoRecord()}: {}) + ); + const err = this.__push( + objname === undefined ? extRecord : [obj[objname], extRecord] + , push); + if(err){ + return err; + } + }else { + const err = this.__push( + objname === undefined ? obj : [obj[objname], obj] + , push); + if(err){ + return err; + } } + // Without columns, records are array }else { - const err = this.__push( - objname === undefined ? record : [record[objname], record] - ); - if(err){ - return err; + if(raw === true || info === true){ + const extRecord = Object.assign( + {record: record}, + raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}, + info === true ? {info: this.__infoRecord()}: {} + ); + const err = this.__push( + objname === undefined ? extRecord : [record[objname], extRecord] + , push); + if(err){ + return err; + } + }else { + const err = this.__push( + objname === undefined ? record : [record[objname], record] + , push); + if(err){ + return err; + } } } } - } - this.__resetRecord(); - } - __firstLineToColumns(record){ - const {firstLineToHeaders} = this.state; - try{ - const headers = firstLineToHeaders === undefined ? record : firstLineToHeaders.call(null, record); - if(!Array.isArray(headers)){ - return this.__error( - new CsvError('CSV_INVALID_COLUMN_MAPPING', [ - 'Invalid Column Mapping:', - 'expect an array from column function,', - `got ${JSON.stringify(headers)}` - ], this.options, this.__infoField(), { - headers: headers, - }) - ); - } - const normalizedHeaders = normalizeColumnsArray(headers); - this.state.expectedRecordLength = normalizedHeaders.length; - this.options.columns = normalizedHeaders; this.__resetRecord(); - return; - }catch(err){ - return err; - } - } - __resetRecord(){ - if(this.options.raw === true){ - this.state.rawBuffer.reset(); - } - this.state.error = undefined; - this.state.record = []; - this.state.record_length = 0; - } - __onField(){ - const {cast, encoding, rtrim, max_record_size} = this.options; - const {enabled, wasQuoting} = this.state; - // Short circuit for the from_line options - if(enabled === false){ - return this.__resetField(); - } - let field = this.state.field.toString(encoding); - if(rtrim === true && wasQuoting === false){ - field = field.trimRight(); - } - if(cast === true){ - const [err, f] = this.__cast(field); - if(err !== undefined) return err; - field = f; - } - this.state.record.push(field); - // Increment record length if record size must not exceed a limit - if(max_record_size !== 0 && typeof field === 'string'){ - this.state.record_length += field.length; - } - this.__resetField(); - } - __resetField(){ - this.state.field.reset(); - this.state.wasQuoting = false; - } - __push(record){ - const {on_record} = this.options; - if(on_record !== undefined){ - const info = this.__infoRecord(); + }, + __firstLineToColumns: function(record){ + const {firstLineToHeaders} = this.state; try{ - record = on_record.call(null, record, info); + const headers = firstLineToHeaders === undefined ? record : firstLineToHeaders.call(null, record); + if(!Array.isArray(headers)){ + return this.__error( + new CsvError('CSV_INVALID_COLUMN_MAPPING', [ + 'Invalid Column Mapping:', + 'expect an array from column function,', + `got ${JSON.stringify(headers)}` + ], this.options, this.__infoField(), { + headers: headers, + }) + ); + } + const normalizedHeaders = normalize_columns_array(headers); + this.state.expectedRecordLength = normalizedHeaders.length; + this.options.columns = normalizedHeaders; + this.__resetRecord(); + return; }catch(err){ return err; } - if(record === undefined || record === null){ return; } - } - this.push(record); - } - // Return a tuple with the error and the casted value - __cast(field){ - const {columns, relax_column_count} = this.options; - const isColumns = Array.isArray(columns); - // Dont loose time calling cast - // because the final record is an object - // and this field can't be associated to a key present in columns - if(isColumns === true && relax_column_count && this.options.columns.length <= this.state.record.length){ - return [undefined, undefined]; - } - if(this.state.castField !== null){ - try{ + }, + __resetRecord: function(){ + if(this.options.raw === true){ + this.state.rawBuffer.reset(); + } + this.state.error = undefined; + this.state.record = []; + this.state.record_length = 0; + }, + __onField: function(){ + const {cast, encoding, rtrim, max_record_size} = this.options; + const {enabled, wasQuoting} = this.state; + // Short circuit for the from_line options + if(enabled === false){ + return this.__resetField(); + } + let field = this.state.field.toString(encoding); + if(rtrim === true && wasQuoting === false){ + field = field.trimRight(); + } + if(cast === true){ + const [err, f] = this.__cast(field); + if(err !== undefined) return err; + field = f; + } + this.state.record.push(field); + // Increment record length if record size must not exceed a limit + if(max_record_size !== 0 && typeof field === 'string'){ + this.state.record_length += field.length; + } + this.__resetField(); + }, + __resetField: function(){ + this.state.field.reset(); + this.state.wasQuoting = false; + }, + __push: function(record, push){ + const {on_record} = this.options; + if(on_record !== undefined){ + const info = this.__infoRecord(); + try{ + record = on_record.call(null, record, info); + }catch(err){ + return err; + } + if(record === undefined || record === null){ return; } + } + push(record); + }, + // Return a tuple with the error and the casted value + __cast: function(field){ + const {columns, relax_column_count} = this.options; + const isColumns = Array.isArray(columns); + // Dont loose time calling cast + // because the final record is an object + // and this field can't be associated to a key present in columns + if(isColumns === true && relax_column_count && this.options.columns.length <= this.state.record.length){ + return [undefined, undefined]; + } + if(this.state.castField !== null){ + try{ + const info = this.__infoField(); + return [undefined, this.state.castField.call(null, field, info)]; + }catch(err){ + return [err]; + } + } + if(this.__isFloat(field)){ + return [undefined, parseFloat(field)]; + }else if(this.options.cast_date !== false){ const info = this.__infoField(); - return [undefined, this.state.castField.call(null, field, info)]; - }catch(err){ - return [err]; + return [undefined, this.options.cast_date.call(null, field, info)]; + } + return [undefined, field]; + }, + // Helper to test if a character is a space or a line delimiter + __isCharTrimable: function(chr){ + return chr === space || chr === tab || chr === cr || chr === nl || chr === np; + }, + // Keep it in case we implement the `cast_int` option + // __isInt(value){ + // // return Number.isInteger(parseInt(value)) + // // return !isNaN( parseInt( obj ) ); + // return /^(\-|\+)?[1-9][0-9]*$/.test(value) + // } + __isFloat: function(value){ + return (value - parseFloat(value) + 1) >= 0; // Borrowed from jquery + }, + __compareBytes: function(sourceBuf, targetBuf, targetPos, firstByte){ + if(sourceBuf[0] !== firstByte) return 0; + const sourceLength = sourceBuf.length; + for(let i = 1; i < sourceLength; i++){ + if(sourceBuf[i] !== targetBuf[targetPos+i]) return 0; + } + return sourceLength; + }, + __isDelimiter: function(buf, pos, chr){ + const {delimiter, ignore_last_delimiters} = this.options; + if(ignore_last_delimiters === true && this.state.record.length === this.options.columns.length - 1){ + return 0; + }else if(ignore_last_delimiters !== false && typeof ignore_last_delimiters === 'number' && this.state.record.length === ignore_last_delimiters - 1){ + return 0; + } + loop1: for(let i = 0; i < delimiter.length; i++){ + const del = delimiter[i]; + if(del[0] === chr){ + for(let j = 1; j < del.length; j++){ + if(del[j] !== buf[pos+j]) continue loop1; + } + return del.length; + } } - } - if(this.__isFloat(field)){ - return [undefined, parseFloat(field)]; - }else if(this.options.cast_date !== false){ - const info = this.__infoField(); - return [undefined, this.options.cast_date.call(null, field, info)]; - } - return [undefined, field]; - } - // Helper to test if a character is a space or a line delimiter - __isCharTrimable(chr){ - return chr === space || chr === tab || chr === cr || chr === nl || chr === np; - } - // Keep it in case we implement the `cast_int` option - // __isInt(value){ - // // return Number.isInteger(parseInt(value)) - // // return !isNaN( parseInt( obj ) ); - // return /^(\-|\+)?[1-9][0-9]*$/.test(value) - // } - __isFloat(value){ - return (value - parseFloat(value) + 1) >= 0; // Borrowed from jquery - } - __compareBytes(sourceBuf, targetBuf, targetPos, firstByte){ - if(sourceBuf[0] !== firstByte) return 0; - const sourceLength = sourceBuf.length; - for(let i = 1; i < sourceLength; i++){ - if(sourceBuf[i] !== targetBuf[targetPos+i]) return 0; - } - return sourceLength; - } - __needMoreData(i, bufLen, end){ - if(end) return false; - const {quote} = this.options; - const {quoting, needMoreDataSize, recordDelimiterMaxLength} = this.state; - const numOfCharLeft = bufLen - i - 1; - const requiredLength = Math.max( - needMoreDataSize, - // Skip if the remaining buffer smaller than record delimiter - recordDelimiterMaxLength, - // Skip if the remaining buffer can be record delimiter following the closing quote - // 1 is for quote.length - quoting ? (quote.length + recordDelimiterMaxLength) : 0, - ); - return numOfCharLeft < requiredLength; - } - __isDelimiter(buf, pos, chr){ - const {delimiter, ignore_last_delimiters} = this.options; - if(ignore_last_delimiters === true && this.state.record.length === this.options.columns.length - 1){ - return 0; - }else if(ignore_last_delimiters !== false && typeof ignore_last_delimiters === 'number' && this.state.record.length === ignore_last_delimiters - 1){ return 0; - } - loop1: for(let i = 0; i < delimiter.length; i++){ - const del = delimiter[i]; - if(del[0] === chr){ - for(let j = 1; j < del.length; j++){ - if(del[j] !== buf[pos+j]) continue loop1; + }, + __isRecordDelimiter: function(chr, buf, pos){ + const {record_delimiter} = this.options; + const recordDelimiterLength = record_delimiter.length; + loop1: for(let i = 0; i < recordDelimiterLength; i++){ + const rd = record_delimiter[i]; + const rdLength = rd.length; + if(rd[0] !== chr){ + continue; } - return del.length; - } - } - return 0; - } - __isRecordDelimiter(chr, buf, pos){ - const {record_delimiter} = this.options; - const recordDelimiterLength = record_delimiter.length; - loop1: for(let i = 0; i < recordDelimiterLength; i++){ - const rd = record_delimiter[i]; - const rdLength = rd.length; - if(rd[0] !== chr){ - continue; - } - for(let j = 1; j < rdLength; j++){ - if(rd[j] !== buf[pos+j]){ - continue loop1; + for(let j = 1; j < rdLength; j++){ + if(rd[j] !== buf[pos+j]){ + continue loop1; + } } + return rd.length; } - return rd.length; - } - return 0; - } - __isEscape(buf, pos, chr){ - const {escape} = this.options; - if(escape === null) return false; - const l = escape.length; - if(escape[0] === chr){ + return 0; + }, + __isEscape: function(buf, pos, chr){ + const {escape} = this.options; + if(escape === null) return false; + const l = escape.length; + if(escape[0] === chr){ + for(let i = 0; i < l; i++){ + if(escape[i] !== buf[pos+i]){ + return false; + } + } + return true; + } + return false; + }, + __isQuote: function(buf, pos){ + const {quote} = this.options; + if(quote === null) return false; + const l = quote.length; for(let i = 0; i < l; i++){ - if(escape[i] !== buf[pos+i]){ + if(quote[i] !== buf[pos+i]){ return false; } } return true; - } - return false; - } - __isQuote(buf, pos){ - const {quote} = this.options; - if(quote === null) return false; - const l = quote.length; - for(let i = 0; i < l; i++){ - if(quote[i] !== buf[pos+i]){ - return false; - } - } - return true; - } - __autoDiscoverRecordDelimiter(buf, pos){ - const {encoding} = this.options; - const chr = buf[pos]; - if(chr === cr){ - if(buf[pos+1] === nl){ - this.options.record_delimiter.push(Buffer.from('\r\n', encoding)); - this.state.recordDelimiterMaxLength = 2; - return 2; - }else { - this.options.record_delimiter.push(Buffer.from('\r', encoding)); + }, + __autoDiscoverRecordDelimiter: function(buf, pos){ + const {encoding} = this.options; + const chr = buf[pos]; + if(chr === cr){ + if(buf[pos+1] === nl){ + this.options.record_delimiter.push(Buffer.from('\r\n', encoding)); + this.state.recordDelimiterMaxLength = 2; + return 2; + }else { + this.options.record_delimiter.push(Buffer.from('\r', encoding)); + this.state.recordDelimiterMaxLength = 1; + return 1; + } + }else if(chr === nl){ + this.options.record_delimiter.push(Buffer.from('\n', encoding)); this.state.recordDelimiterMaxLength = 1; return 1; } - }else if(chr === nl){ - this.options.record_delimiter.push(Buffer.from('\n', encoding)); - this.state.recordDelimiterMaxLength = 1; - return 1; - } - return 0; - } - __error(msg){ - const {encoding, raw, skip_records_with_error} = this.options; - const err = typeof msg === 'string' ? new Error(msg) : msg; - if(skip_records_with_error){ - this.state.recordHasError = true; - this.emit('skip', err, raw ? this.state.rawBuffer.toString(encoding) : undefined); - return undefined; - }else { - return err; + return 0; + }, + __error: function(msg){ + const {encoding, raw, skip_records_with_error} = this.options; + const err = typeof msg === 'string' ? new Error(msg) : msg; + if(skip_records_with_error){ + this.state.recordHasError = true; + if(this.options.on_skip !== undefined){ + this.options.on_skip(err, raw ? this.state.rawBuffer.toString(encoding) : undefined); + } + // this.emit('skip', err, raw ? this.state.rawBuffer.toString(encoding) : undefined); + return undefined; + }else { + return err; + } + }, + __infoDataSet: function(){ + return { + ...this.info, + columns: this.options.columns + }; + }, + __infoRecord: function(){ + const {columns, raw, encoding} = this.options; + return { + ...this.__infoDataSet(), + error: this.state.error, + header: columns === true, + index: this.state.record.length, + raw: raw ? this.state.rawBuffer.toString(encoding) : undefined + }; + }, + __infoField: function(){ + const {columns} = this.options; + const isColumns = Array.isArray(columns); + return { + ...this.__infoRecord(), + column: isColumns === true ? + (columns.length > this.state.record.length ? + columns[this.state.record.length].name : + null + ) : + this.state.record.length, + quoting: this.state.wasQuoting, + }; } - } - __infoDataSet(){ - return { - ...this.info, - columns: this.options.columns - }; - } - __infoRecord(){ - const {columns, raw, encoding} = this.options; - return { - ...this.__infoDataSet(), - error: this.state.error, - header: columns === true, - index: this.state.record.length, - raw: raw ? this.state.rawBuffer.toString(encoding) : undefined - }; - } - __infoField(){ - const {columns} = this.options; - const isColumns = Array.isArray(columns); - return { - ...this.__infoRecord(), - column: isColumns === true ? - (columns.length > this.state.record.length ? - columns[this.state.record.length].name : - null - ) : - this.state.record.length, - quoting: this.state.wasQuoting, - }; - } - } + }; + }; - const parse = function(data, options={}){ + const parse = function(data, opts={}){ if(typeof data === 'string'){ data = Buffer.from(data); } - const records = options && options.objname ? {} : []; - const parser = new Parser(options); - parser.push = function(record){ - if(record === null){ - return; - } - if(options.objname === undefined) + const records = opts && opts.objname ? {} : []; + const parser = transform(opts); + const push = (record) => { + if(parser.options.objname === undefined) records.push(record); else { records[record[0]] = record[1]; } }; - const err1 = parser.__parse(data, false); + const close = () => {}; + const err1 = parser.parse(data, false, push, close); if(err1 !== undefined) throw err1; - const err2 = parser.__parse(undefined, true); + const err2 = parser.parse(undefined, true, push, close); if(err2 !== undefined) throw err2; return records; }; diff --git a/packages/csv-parse/dist/umd/index.js b/packages/csv-parse/dist/umd/index.js index 9cfcf4440..5223def8f 100644 --- a/packages/csv-parse/dist/umd/index.js +++ b/packages/csv-parse/dist/umd/index.js @@ -2633,7 +2633,7 @@ } }); for (var x = args[i]; i < len; x = args[++i]) { - if (isNull(x) || !isObject$1(x)) { + if (isNull(x) || !isObject(x)) { str += ' ' + x; } else { str += ' ' + inspect(x); @@ -3041,19 +3041,19 @@ } function isRegExp(re) { - return isObject$1(re) && objectToString(re) === '[object RegExp]'; + return isObject(re) && objectToString(re) === '[object RegExp]'; } - function isObject$1(arg) { + function isObject(arg) { return typeof arg === 'object' && arg !== null; } function isDate(d) { - return isObject$1(d) && objectToString(d) === '[object Date]'; + return isObject(d) && objectToString(d) === '[object Date]'; } function isError(e) { - return isObject$1(e) && + return isObject(e) && (objectToString(e) === '[object Error]' || e instanceof Error); } @@ -3067,7 +3067,7 @@ function _extend(origin, add) { // Don't do anything if add isn't an object - if (!add || !isObject$1(add)) return origin; + if (!add || !isObject(add)) return origin; var keys = Object.keys(add); var i = keys.length; @@ -4972,6 +4972,55 @@ return dest; }; + const is_object = function(obj){ + return (typeof obj === 'object' && obj !== null && !Array.isArray(obj)); + }; + + class CsvError extends Error { + constructor(code, message, options, ...contexts) { + if(Array.isArray(message)) message = message.join(' '); + super(message); + if(Error.captureStackTrace !== undefined){ + Error.captureStackTrace(this, CsvError); + } + this.code = code; + for(const context of contexts){ + for(const key in context){ + const value = context[key]; + this[key] = isBuffer(value) ? value.toString(options.encoding) : value == null ? value : JSON.parse(JSON.stringify(value)); + } + } + } + } + + const normalize_columns_array = function(columns){ + const normalizedColumns = []; + for(let i = 0, l = columns.length; i < l; i++){ + const column = columns[i]; + if(column === undefined || column === null || column === false){ + normalizedColumns[i] = { disabled: true }; + }else if(typeof column === 'string'){ + normalizedColumns[i] = { name: column }; + }else if(is_object(column)){ + if(typeof column.name !== 'string'){ + throw new CsvError('CSV_OPTION_COLUMNS_MISSING_NAME', [ + 'Option columns missing name:', + `property "name" is required at position ${i}`, + 'when column is an object literal' + ]); + } + normalizedColumns[i] = column; + }else { + throw new CsvError('CSV_INVALID_COLUMN_DEFINITION', [ + 'Invalid column definition:', + 'expect a string or a literal object,', + `got ${JSON.stringify(column)} at position ${i}` + ]); + } + } + return normalizedColumns; + }; + class ResizeableBuffer{ constructor(size=100){ this.size = size; @@ -5034,1210 +5083,1192 @@ } } - // white space characters - // https://en.wikipedia.org/wiki/Whitespace_character - // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions/Character_Classes#Types - // \f\n\r\t\v\u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff - const tab = 9; - const nl = 10; // \n, 0x0A in hexadecimal, 10 in decimal - const np = 12; - const cr = 13; // \r, 0x0D in hexadécimal, 13 in decimal - const space = 32; - const boms = { - // Note, the following are equals: - // Buffer.from("\ufeff") - // Buffer.from([239, 187, 191]) - // Buffer.from('EFBBBF', 'hex') - 'utf8': Buffer.from([239, 187, 191]), - // Note, the following are equals: - // Buffer.from "\ufeff", 'utf16le - // Buffer.from([255, 254]) - 'utf16le': Buffer.from([255, 254]) + const init_state = function(options){ + return { + bomSkipped: false, + bufBytesStart: 0, + castField: options.cast_function, + commenting: false, + // Current error encountered by a record + error: undefined, + enabled: options.from_line === 1, + escaping: false, + escapeIsQuote: isBuffer(options.escape) && isBuffer(options.quote) && Buffer.compare(options.escape, options.quote) === 0, + // columns can be `false`, `true`, `Array` + expectedRecordLength: Array.isArray(options.columns) ? options.columns.length : undefined, + field: new ResizeableBuffer(20), + firstLineToHeaders: options.cast_first_line_to_header, + needMoreDataSize: Math.max( + // Skip if the remaining buffer smaller than comment + options.comment !== null ? options.comment.length : 0, + // Skip if the remaining buffer can be delimiter + ...options.delimiter.map((delimiter) => delimiter.length), + // Skip if the remaining buffer can be escape sequence + options.quote !== null ? options.quote.length : 0, + ), + previousBuf: undefined, + quoting: false, + stop: false, + rawBuffer: new ResizeableBuffer(100), + record: [], + recordHasError: false, + record_length: 0, + recordDelimiterMaxLength: options.record_delimiter.length === 0 ? 2 : Math.max(...options.record_delimiter.map((v) => v.length)), + trimChars: [Buffer.from(' ', options.encoding)[0], Buffer.from('\t', options.encoding)[0]], + wasQuoting: false, + wasRowDelimiter: false + }; }; - class CsvError extends Error { - constructor(code, message, options, ...contexts) { - if(Array.isArray(message)) message = message.join(' '); - super(message); - if(Error.captureStackTrace !== undefined){ - Error.captureStackTrace(this, CsvError); - } - this.code = code; - for(const context of contexts){ - for(const key in context){ - const value = context[key]; - this[key] = isBuffer(value) ? value.toString(options.encoding) : value == null ? value : JSON.parse(JSON.stringify(value)); - } - } - } - } - const underscore = function(str){ return str.replace(/([A-Z])/g, function(_, match){ return '_' + match.toLowerCase(); }); }; - const isObject = function(obj){ - return (typeof obj === 'object' && obj !== null && !Array.isArray(obj)); - }; - - const isRecordEmpty = function(record){ - return record.every((field) => field == null || field.toString && field.toString().trim() === ''); - }; - - const normalizeColumnsArray = function(columns){ - const normalizedColumns = []; - for(let i = 0, l = columns.length; i < l; i++){ - const column = columns[i]; - if(column === undefined || column === null || column === false){ - normalizedColumns[i] = { disabled: true }; - }else if(typeof column === 'string'){ - normalizedColumns[i] = { name: column }; - }else if(isObject(column)){ - if(typeof column.name !== 'string'){ - throw new CsvError('CSV_OPTION_COLUMNS_MISSING_NAME', [ - 'Option columns missing name:', - `property "name" is required at position ${i}`, - 'when column is an object literal' - ]); - } - normalizedColumns[i] = column; - }else { - throw new CsvError('CSV_INVALID_COLUMN_DEFINITION', [ - 'Invalid column definition:', - 'expect a string or a literal object,', - `got ${JSON.stringify(column)} at position ${i}` - ]); - } - } - return normalizedColumns; - }; - - class Parser extends Transform { - constructor(opts = {}){ - super({...{readableObjectMode: true}, ...opts, encoding: null}); - this.__originalOptions = opts; - this.__normalizeOptions(opts); - } - __normalizeOptions(opts){ - const options = {}; - // Merge with user options - for(const opt in opts){ - options[underscore(opt)] = opts[opt]; - } - // Normalize option `encoding` - // Note: defined first because other options depends on it - // to convert chars/strings into buffers. - if(options.encoding === undefined || options.encoding === true){ - options.encoding = 'utf8'; - }else if(options.encoding === null || options.encoding === false){ - options.encoding = null; - }else if(typeof options.encoding !== 'string' && options.encoding !== null){ - throw new CsvError('CSV_INVALID_OPTION_ENCODING', [ - 'Invalid option encoding:', - 'encoding must be a string or null to return a buffer,', - `got ${JSON.stringify(options.encoding)}` - ], options); - } - // Normalize option `bom` - if(options.bom === undefined || options.bom === null || options.bom === false){ - options.bom = false; - }else if(options.bom !== true){ - throw new CsvError('CSV_INVALID_OPTION_BOM', [ - 'Invalid option bom:', 'bom must be true,', - `got ${JSON.stringify(options.bom)}` - ], options); - } - // Normalize option `cast` - let fnCastField = null; - if(options.cast === undefined || options.cast === null || options.cast === false || options.cast === ''){ - options.cast = undefined; - }else if(typeof options.cast === 'function'){ - fnCastField = options.cast; - options.cast = true; - }else if(options.cast !== true){ - throw new CsvError('CSV_INVALID_OPTION_CAST', [ - 'Invalid option cast:', 'cast must be true or a function,', - `got ${JSON.stringify(options.cast)}` - ], options); - } - // Normalize option `cast_date` - if(options.cast_date === undefined || options.cast_date === null || options.cast_date === false || options.cast_date === ''){ - options.cast_date = false; - }else if(options.cast_date === true){ - options.cast_date = function(value){ - const date = Date.parse(value); - return !isNaN(date) ? new Date(date) : value; - }; - }else { - throw new CsvError('CSV_INVALID_OPTION_CAST_DATE', [ - 'Invalid option cast_date:', 'cast_date must be true or a function,', - `got ${JSON.stringify(options.cast_date)}` + const normalize_options = function(opts){ + const options = {}; + // Merge with user options + for(const opt in opts){ + options[underscore(opt)] = opts[opt]; + } + // Normalize option `encoding` + // Note: defined first because other options depends on it + // to convert chars/strings into buffers. + if(options.encoding === undefined || options.encoding === true){ + options.encoding = 'utf8'; + }else if(options.encoding === null || options.encoding === false){ + options.encoding = null; + }else if(typeof options.encoding !== 'string' && options.encoding !== null){ + throw new CsvError('CSV_INVALID_OPTION_ENCODING', [ + 'Invalid option encoding:', + 'encoding must be a string or null to return a buffer,', + `got ${JSON.stringify(options.encoding)}` + ], options); + } + // Normalize option `bom` + if(options.bom === undefined || options.bom === null || options.bom === false){ + options.bom = false; + }else if(options.bom !== true){ + throw new CsvError('CSV_INVALID_OPTION_BOM', [ + 'Invalid option bom:', 'bom must be true,', + `got ${JSON.stringify(options.bom)}` + ], options); + } + // Normalize option `cast` + options.cast_function = null; + if(options.cast === undefined || options.cast === null || options.cast === false || options.cast === ''){ + options.cast = undefined; + }else if(typeof options.cast === 'function'){ + options.cast_function = options.cast; + options.cast = true; + }else if(options.cast !== true){ + throw new CsvError('CSV_INVALID_OPTION_CAST', [ + 'Invalid option cast:', 'cast must be true or a function,', + `got ${JSON.stringify(options.cast)}` + ], options); + } + // Normalize option `cast_date` + if(options.cast_date === undefined || options.cast_date === null || options.cast_date === false || options.cast_date === ''){ + options.cast_date = false; + }else if(options.cast_date === true){ + options.cast_date = function(value){ + const date = Date.parse(value); + return !isNaN(date) ? new Date(date) : value; + }; + }else { + throw new CsvError('CSV_INVALID_OPTION_CAST_DATE', [ + 'Invalid option cast_date:', 'cast_date must be true or a function,', + `got ${JSON.stringify(options.cast_date)}` + ], options); + } + // Normalize option `columns` + options.cast_first_line_to_header = null; + if(options.columns === true){ + // Fields in the first line are converted as-is to columns + options.cast_first_line_to_header = undefined; + }else if(typeof options.columns === 'function'){ + options.cast_first_line_to_header = options.columns; + options.columns = true; + }else if(Array.isArray(options.columns)){ + options.columns = normalize_columns_array(options.columns); + }else if(options.columns === undefined || options.columns === null || options.columns === false){ + options.columns = false; + }else { + throw new CsvError('CSV_INVALID_OPTION_COLUMNS', [ + 'Invalid option columns:', + 'expect an array, a function or true,', + `got ${JSON.stringify(options.columns)}` + ], options); + } + // Normalize option `group_columns_by_name` + if(options.group_columns_by_name === undefined || options.group_columns_by_name === null || options.group_columns_by_name === false){ + options.group_columns_by_name = false; + }else if(options.group_columns_by_name !== true){ + throw new CsvError('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ + 'Invalid option group_columns_by_name:', + 'expect an boolean,', + `got ${JSON.stringify(options.group_columns_by_name)}` + ], options); + }else if(options.columns === false){ + throw new CsvError('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ + 'Invalid option group_columns_by_name:', + 'the `columns` mode must be activated.' + ], options); + } + // Normalize option `comment` + if(options.comment === undefined || options.comment === null || options.comment === false || options.comment === ''){ + options.comment = null; + }else { + if(typeof options.comment === 'string'){ + options.comment = Buffer.from(options.comment, options.encoding); + } + if(!isBuffer(options.comment)){ + throw new CsvError('CSV_INVALID_OPTION_COMMENT', [ + 'Invalid option comment:', + 'comment must be a buffer or a string,', + `got ${JSON.stringify(options.comment)}` ], options); } - // Normalize option `columns` - let fnFirstLineToHeaders = null; - if(options.columns === true){ - // Fields in the first line are converted as-is to columns - fnFirstLineToHeaders = undefined; - }else if(typeof options.columns === 'function'){ - fnFirstLineToHeaders = options.columns; - options.columns = true; - }else if(Array.isArray(options.columns)){ - options.columns = normalizeColumnsArray(options.columns); - }else if(options.columns === undefined || options.columns === null || options.columns === false){ - options.columns = false; - }else { - throw new CsvError('CSV_INVALID_OPTION_COLUMNS', [ - 'Invalid option columns:', - 'expect an array, a function or true,', - `got ${JSON.stringify(options.columns)}` - ], options); + } + // Normalize option `delimiter` + const delimiter_json = JSON.stringify(options.delimiter); + if(!Array.isArray(options.delimiter)) options.delimiter = [options.delimiter]; + if(options.delimiter.length === 0){ + throw new CsvError('CSV_INVALID_OPTION_DELIMITER', [ + 'Invalid option delimiter:', + 'delimiter must be a non empty string or buffer or array of string|buffer,', + `got ${delimiter_json}` + ], options); + } + options.delimiter = options.delimiter.map(function(delimiter){ + if(delimiter === undefined || delimiter === null || delimiter === false){ + return Buffer.from(',', options.encoding); } - // Normalize option `group_columns_by_name` - if(options.group_columns_by_name === undefined || options.group_columns_by_name === null || options.group_columns_by_name === false){ - options.group_columns_by_name = false; - }else if(options.group_columns_by_name !== true){ - throw new CsvError('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ - 'Invalid option group_columns_by_name:', - 'expect an boolean,', - `got ${JSON.stringify(options.group_columns_by_name)}` - ], options); - }else if(options.columns === false){ - throw new CsvError('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ - 'Invalid option group_columns_by_name:', - 'the `columns` mode must be activated.' - ], options); + if(typeof delimiter === 'string'){ + delimiter = Buffer.from(delimiter, options.encoding); } - // Normalize option `comment` - if(options.comment === undefined || options.comment === null || options.comment === false || options.comment === ''){ - options.comment = null; - }else { - if(typeof options.comment === 'string'){ - options.comment = Buffer.from(options.comment, options.encoding); - } - if(!isBuffer(options.comment)){ - throw new CsvError('CSV_INVALID_OPTION_COMMENT', [ - 'Invalid option comment:', - 'comment must be a buffer or a string,', - `got ${JSON.stringify(options.comment)}` - ], options); - } - } - // Normalize option `delimiter` - const delimiter_json = JSON.stringify(options.delimiter); - if(!Array.isArray(options.delimiter)) options.delimiter = [options.delimiter]; - if(options.delimiter.length === 0){ + if(!isBuffer(delimiter) || delimiter.length === 0){ throw new CsvError('CSV_INVALID_OPTION_DELIMITER', [ 'Invalid option delimiter:', 'delimiter must be a non empty string or buffer or array of string|buffer,', `got ${delimiter_json}` ], options); } - options.delimiter = options.delimiter.map(function(delimiter){ - if(delimiter === undefined || delimiter === null || delimiter === false){ - return Buffer.from(',', options.encoding); - } - if(typeof delimiter === 'string'){ - delimiter = Buffer.from(delimiter, options.encoding); - } - if(!isBuffer(delimiter) || delimiter.length === 0){ - throw new CsvError('CSV_INVALID_OPTION_DELIMITER', [ - 'Invalid option delimiter:', - 'delimiter must be a non empty string or buffer or array of string|buffer,', - `got ${delimiter_json}` - ], options); - } - return delimiter; - }); - // Normalize option `escape` - if(options.escape === undefined || options.escape === true){ - options.escape = Buffer.from('"', options.encoding); - }else if(typeof options.escape === 'string'){ - options.escape = Buffer.from(options.escape, options.encoding); - }else if (options.escape === null || options.escape === false){ - options.escape = null; - } - if(options.escape !== null){ - if(!isBuffer(options.escape)){ - throw new Error(`Invalid Option: escape must be a buffer, a string or a boolean, got ${JSON.stringify(options.escape)}`); + return delimiter; + }); + // Normalize option `escape` + if(options.escape === undefined || options.escape === true){ + options.escape = Buffer.from('"', options.encoding); + }else if(typeof options.escape === 'string'){ + options.escape = Buffer.from(options.escape, options.encoding); + }else if (options.escape === null || options.escape === false){ + options.escape = null; + } + if(options.escape !== null){ + if(!isBuffer(options.escape)){ + throw new Error(`Invalid Option: escape must be a buffer, a string or a boolean, got ${JSON.stringify(options.escape)}`); + } + } + // Normalize option `from` + if(options.from === undefined || options.from === null){ + options.from = 1; + }else { + if(typeof options.from === 'string' && /\d+/.test(options.from)){ + options.from = parseInt(options.from); + } + if(Number.isInteger(options.from)){ + if(options.from < 0){ + throw new Error(`Invalid Option: from must be a positive integer, got ${JSON.stringify(opts.from)}`); } - } - // Normalize option `from` - if(options.from === undefined || options.from === null){ - options.from = 1; }else { - if(typeof options.from === 'string' && /\d+/.test(options.from)){ - options.from = parseInt(options.from); - } - if(Number.isInteger(options.from)){ - if(options.from < 0){ - throw new Error(`Invalid Option: from must be a positive integer, got ${JSON.stringify(opts.from)}`); - } - }else { - throw new Error(`Invalid Option: from must be an integer, got ${JSON.stringify(options.from)}`); - } + throw new Error(`Invalid Option: from must be an integer, got ${JSON.stringify(options.from)}`); } - // Normalize option `from_line` - if(options.from_line === undefined || options.from_line === null){ - options.from_line = 1; - }else { - if(typeof options.from_line === 'string' && /\d+/.test(options.from_line)){ - options.from_line = parseInt(options.from_line); - } - if(Number.isInteger(options.from_line)){ - if(options.from_line <= 0){ - throw new Error(`Invalid Option: from_line must be a positive integer greater than 0, got ${JSON.stringify(opts.from_line)}`); - } - }else { - throw new Error(`Invalid Option: from_line must be an integer, got ${JSON.stringify(opts.from_line)}`); - } + } + // Normalize option `from_line` + if(options.from_line === undefined || options.from_line === null){ + options.from_line = 1; + }else { + if(typeof options.from_line === 'string' && /\d+/.test(options.from_line)){ + options.from_line = parseInt(options.from_line); } - // Normalize options `ignore_last_delimiters` - if(options.ignore_last_delimiters === undefined || options.ignore_last_delimiters === null){ - options.ignore_last_delimiters = false; - }else if(typeof options.ignore_last_delimiters === 'number'){ - options.ignore_last_delimiters = Math.floor(options.ignore_last_delimiters); - if(options.ignore_last_delimiters === 0){ - options.ignore_last_delimiters = false; + if(Number.isInteger(options.from_line)){ + if(options.from_line <= 0){ + throw new Error(`Invalid Option: from_line must be a positive integer greater than 0, got ${JSON.stringify(opts.from_line)}`); } - }else if(typeof options.ignore_last_delimiters !== 'boolean'){ - throw new CsvError('CSV_INVALID_OPTION_IGNORE_LAST_DELIMITERS', [ - 'Invalid option `ignore_last_delimiters`:', - 'the value must be a boolean value or an integer,', - `got ${JSON.stringify(options.ignore_last_delimiters)}` - ], options); - } - if(options.ignore_last_delimiters === true && options.columns === false){ - throw new CsvError('CSV_IGNORE_LAST_DELIMITERS_REQUIRES_COLUMNS', [ - 'The option `ignore_last_delimiters`', - 'requires the activation of the `columns` option' - ], options); - } - // Normalize option `info` - if(options.info === undefined || options.info === null || options.info === false){ - options.info = false; - }else if(options.info !== true){ - throw new Error(`Invalid Option: info must be true, got ${JSON.stringify(options.info)}`); - } - // Normalize option `max_record_size` - if(options.max_record_size === undefined || options.max_record_size === null || options.max_record_size === false){ - options.max_record_size = 0; - }else if(Number.isInteger(options.max_record_size) && options.max_record_size >= 0);else if(typeof options.max_record_size === 'string' && /\d+/.test(options.max_record_size)){ - options.max_record_size = parseInt(options.max_record_size); }else { - throw new Error(`Invalid Option: max_record_size must be a positive integer, got ${JSON.stringify(options.max_record_size)}`); - } - // Normalize option `objname` - if(options.objname === undefined || options.objname === null || options.objname === false){ - options.objname = undefined; - }else if(isBuffer(options.objname)){ - if(options.objname.length === 0){ - throw new Error(`Invalid Option: objname must be a non empty buffer`); - } - if(options.encoding === null);else { - options.objname = options.objname.toString(options.encoding); - } - }else if(typeof options.objname === 'string'){ - if(options.objname.length === 0){ - throw new Error(`Invalid Option: objname must be a non empty string`); - } - // Great, nothing to do - }else if(typeof options.objname === 'number');else { - throw new Error(`Invalid Option: objname must be a string or a buffer, got ${options.objname}`); - } - if(options.objname !== undefined){ - if(typeof options.objname === 'number'){ - if(options.columns !== false){ - throw Error('Invalid Option: objname index cannot be combined with columns or be defined as a field'); - } - }else { // A string or a buffer - if(options.columns === false){ - throw Error('Invalid Option: objname field must be combined with columns or be defined as an index'); - } - } + throw new Error(`Invalid Option: from_line must be an integer, got ${JSON.stringify(opts.from_line)}`); } - // Normalize option `on_record` - if(options.on_record === undefined || options.on_record === null){ - options.on_record = undefined; - }else if(typeof options.on_record !== 'function'){ - throw new CsvError('CSV_INVALID_OPTION_ON_RECORD', [ - 'Invalid option `on_record`:', - 'expect a function,', - `got ${JSON.stringify(options.on_record)}` - ], options); + } + // Normalize options `ignore_last_delimiters` + if(options.ignore_last_delimiters === undefined || options.ignore_last_delimiters === null){ + options.ignore_last_delimiters = false; + }else if(typeof options.ignore_last_delimiters === 'number'){ + options.ignore_last_delimiters = Math.floor(options.ignore_last_delimiters); + if(options.ignore_last_delimiters === 0){ + options.ignore_last_delimiters = false; } - // Normalize option `quote` - if(options.quote === null || options.quote === false || options.quote === ''){ - options.quote = null; - }else { - if(options.quote === undefined || options.quote === true){ - options.quote = Buffer.from('"', options.encoding); - }else if(typeof options.quote === 'string'){ - options.quote = Buffer.from(options.quote, options.encoding); + }else if(typeof options.ignore_last_delimiters !== 'boolean'){ + throw new CsvError('CSV_INVALID_OPTION_IGNORE_LAST_DELIMITERS', [ + 'Invalid option `ignore_last_delimiters`:', + 'the value must be a boolean value or an integer,', + `got ${JSON.stringify(options.ignore_last_delimiters)}` + ], options); + } + if(options.ignore_last_delimiters === true && options.columns === false){ + throw new CsvError('CSV_IGNORE_LAST_DELIMITERS_REQUIRES_COLUMNS', [ + 'The option `ignore_last_delimiters`', + 'requires the activation of the `columns` option' + ], options); + } + // Normalize option `info` + if(options.info === undefined || options.info === null || options.info === false){ + options.info = false; + }else if(options.info !== true){ + throw new Error(`Invalid Option: info must be true, got ${JSON.stringify(options.info)}`); + } + // Normalize option `max_record_size` + if(options.max_record_size === undefined || options.max_record_size === null || options.max_record_size === false){ + options.max_record_size = 0; + }else if(Number.isInteger(options.max_record_size) && options.max_record_size >= 0);else if(typeof options.max_record_size === 'string' && /\d+/.test(options.max_record_size)){ + options.max_record_size = parseInt(options.max_record_size); + }else { + throw new Error(`Invalid Option: max_record_size must be a positive integer, got ${JSON.stringify(options.max_record_size)}`); + } + // Normalize option `objname` + if(options.objname === undefined || options.objname === null || options.objname === false){ + options.objname = undefined; + }else if(isBuffer(options.objname)){ + if(options.objname.length === 0){ + throw new Error(`Invalid Option: objname must be a non empty buffer`); + } + if(options.encoding === null);else { + options.objname = options.objname.toString(options.encoding); + } + }else if(typeof options.objname === 'string'){ + if(options.objname.length === 0){ + throw new Error(`Invalid Option: objname must be a non empty string`); + } + // Great, nothing to do + }else if(typeof options.objname === 'number');else { + throw new Error(`Invalid Option: objname must be a string or a buffer, got ${options.objname}`); + } + if(options.objname !== undefined){ + if(typeof options.objname === 'number'){ + if(options.columns !== false){ + throw Error('Invalid Option: objname index cannot be combined with columns or be defined as a field'); } - if(!isBuffer(options.quote)){ - throw new Error(`Invalid Option: quote must be a buffer or a string, got ${JSON.stringify(options.quote)}`); + }else { // A string or a buffer + if(options.columns === false){ + throw Error('Invalid Option: objname field must be combined with columns or be defined as an index'); } } - // Normalize option `raw` - if(options.raw === undefined || options.raw === null || options.raw === false){ - options.raw = false; - }else if(options.raw !== true){ - throw new Error(`Invalid Option: raw must be true, got ${JSON.stringify(options.raw)}`); - } - // Normalize option `record_delimiter` - if(options.record_delimiter === undefined){ - options.record_delimiter = []; - }else if(typeof options.record_delimiter === 'string' || isBuffer(options.record_delimiter)){ - if(options.record_delimiter.length === 0){ - throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ - 'Invalid option `record_delimiter`:', - 'value must be a non empty string or buffer,', - `got ${JSON.stringify(options.record_delimiter)}` - ], options); - } - options.record_delimiter = [options.record_delimiter]; - }else if(!Array.isArray(options.record_delimiter)){ + } + // Normalize option `on_record` + if(options.on_record === undefined || options.on_record === null){ + options.on_record = undefined; + }else if(typeof options.on_record !== 'function'){ + throw new CsvError('CSV_INVALID_OPTION_ON_RECORD', [ + 'Invalid option `on_record`:', + 'expect a function,', + `got ${JSON.stringify(options.on_record)}` + ], options); + } + // Normalize option `quote` + if(options.quote === null || options.quote === false || options.quote === ''){ + options.quote = null; + }else { + if(options.quote === undefined || options.quote === true){ + options.quote = Buffer.from('"', options.encoding); + }else if(typeof options.quote === 'string'){ + options.quote = Buffer.from(options.quote, options.encoding); + } + if(!isBuffer(options.quote)){ + throw new Error(`Invalid Option: quote must be a buffer or a string, got ${JSON.stringify(options.quote)}`); + } + } + // Normalize option `raw` + if(options.raw === undefined || options.raw === null || options.raw === false){ + options.raw = false; + }else if(options.raw !== true){ + throw new Error(`Invalid Option: raw must be true, got ${JSON.stringify(options.raw)}`); + } + // Normalize option `record_delimiter` + if(options.record_delimiter === undefined){ + options.record_delimiter = []; + }else if(typeof options.record_delimiter === 'string' || isBuffer(options.record_delimiter)){ + if(options.record_delimiter.length === 0){ throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ 'Invalid option `record_delimiter`:', - 'value must be a string, a buffer or array of string|buffer,', + 'value must be a non empty string or buffer,', `got ${JSON.stringify(options.record_delimiter)}` ], options); } - options.record_delimiter = options.record_delimiter.map(function(rd, i){ - if(typeof rd !== 'string' && ! isBuffer(rd)){ - throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ - 'Invalid option `record_delimiter`:', - 'value must be a string, a buffer or array of string|buffer', - `at index ${i},`, - `got ${JSON.stringify(rd)}` - ], options); - }else if(rd.length === 0){ - throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ - 'Invalid option `record_delimiter`:', - 'value must be a non empty string or buffer', - `at index ${i},`, - `got ${JSON.stringify(rd)}` - ], options); - } - if(typeof rd === 'string'){ - rd = Buffer.from(rd, options.encoding); - } - return rd; - }); - // Normalize option `relax_column_count` - if(typeof options.relax_column_count === 'boolean');else if(options.relax_column_count === undefined || options.relax_column_count === null){ - options.relax_column_count = false; - }else { - throw new Error(`Invalid Option: relax_column_count must be a boolean, got ${JSON.stringify(options.relax_column_count)}`); - } - if(typeof options.relax_column_count_less === 'boolean');else if(options.relax_column_count_less === undefined || options.relax_column_count_less === null){ - options.relax_column_count_less = false; - }else { - throw new Error(`Invalid Option: relax_column_count_less must be a boolean, got ${JSON.stringify(options.relax_column_count_less)}`); + options.record_delimiter = [options.record_delimiter]; + }else if(!Array.isArray(options.record_delimiter)){ + throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ + 'Invalid option `record_delimiter`:', + 'value must be a string, a buffer or array of string|buffer,', + `got ${JSON.stringify(options.record_delimiter)}` + ], options); + } + options.record_delimiter = options.record_delimiter.map(function(rd, i){ + if(typeof rd !== 'string' && ! isBuffer(rd)){ + throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ + 'Invalid option `record_delimiter`:', + 'value must be a string, a buffer or array of string|buffer', + `at index ${i},`, + `got ${JSON.stringify(rd)}` + ], options); + }else if(rd.length === 0){ + throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ + 'Invalid option `record_delimiter`:', + 'value must be a non empty string or buffer', + `at index ${i},`, + `got ${JSON.stringify(rd)}` + ], options); } - if(typeof options.relax_column_count_more === 'boolean');else if(options.relax_column_count_more === undefined || options.relax_column_count_more === null){ - options.relax_column_count_more = false; - }else { - throw new Error(`Invalid Option: relax_column_count_more must be a boolean, got ${JSON.stringify(options.relax_column_count_more)}`); + if(typeof rd === 'string'){ + rd = Buffer.from(rd, options.encoding); } - // Normalize option `relax_quotes` - if(typeof options.relax_quotes === 'boolean');else if(options.relax_quotes === undefined || options.relax_quotes === null){ - options.relax_quotes = false; + return rd; + }); + // Normalize option `relax_column_count` + if(typeof options.relax_column_count === 'boolean');else if(options.relax_column_count === undefined || options.relax_column_count === null){ + options.relax_column_count = false; + }else { + throw new Error(`Invalid Option: relax_column_count must be a boolean, got ${JSON.stringify(options.relax_column_count)}`); + } + if(typeof options.relax_column_count_less === 'boolean');else if(options.relax_column_count_less === undefined || options.relax_column_count_less === null){ + options.relax_column_count_less = false; + }else { + throw new Error(`Invalid Option: relax_column_count_less must be a boolean, got ${JSON.stringify(options.relax_column_count_less)}`); + } + if(typeof options.relax_column_count_more === 'boolean');else if(options.relax_column_count_more === undefined || options.relax_column_count_more === null){ + options.relax_column_count_more = false; + }else { + throw new Error(`Invalid Option: relax_column_count_more must be a boolean, got ${JSON.stringify(options.relax_column_count_more)}`); + } + // Normalize option `relax_quotes` + if(typeof options.relax_quotes === 'boolean');else if(options.relax_quotes === undefined || options.relax_quotes === null){ + options.relax_quotes = false; + }else { + throw new Error(`Invalid Option: relax_quotes must be a boolean, got ${JSON.stringify(options.relax_quotes)}`); + } + // Normalize option `skip_empty_lines` + if(typeof options.skip_empty_lines === 'boolean');else if(options.skip_empty_lines === undefined || options.skip_empty_lines === null){ + options.skip_empty_lines = false; + }else { + throw new Error(`Invalid Option: skip_empty_lines must be a boolean, got ${JSON.stringify(options.skip_empty_lines)}`); + } + // Normalize option `skip_records_with_empty_values` + if(typeof options.skip_records_with_empty_values === 'boolean');else if(options.skip_records_with_empty_values === undefined || options.skip_records_with_empty_values === null){ + options.skip_records_with_empty_values = false; + }else { + throw new Error(`Invalid Option: skip_records_with_empty_values must be a boolean, got ${JSON.stringify(options.skip_records_with_empty_values)}`); + } + // Normalize option `skip_records_with_error` + if(typeof options.skip_records_with_error === 'boolean');else if(options.skip_records_with_error === undefined || options.skip_records_with_error === null){ + options.skip_records_with_error = false; + }else { + throw new Error(`Invalid Option: skip_records_with_error must be a boolean, got ${JSON.stringify(options.skip_records_with_error)}`); + } + // Normalize option `rtrim` + if(options.rtrim === undefined || options.rtrim === null || options.rtrim === false){ + options.rtrim = false; + }else if(options.rtrim !== true){ + throw new Error(`Invalid Option: rtrim must be a boolean, got ${JSON.stringify(options.rtrim)}`); + } + // Normalize option `ltrim` + if(options.ltrim === undefined || options.ltrim === null || options.ltrim === false){ + options.ltrim = false; + }else if(options.ltrim !== true){ + throw new Error(`Invalid Option: ltrim must be a boolean, got ${JSON.stringify(options.ltrim)}`); + } + // Normalize option `trim` + if(options.trim === undefined || options.trim === null || options.trim === false){ + options.trim = false; + }else if(options.trim !== true){ + throw new Error(`Invalid Option: trim must be a boolean, got ${JSON.stringify(options.trim)}`); + } + // Normalize options `trim`, `ltrim` and `rtrim` + if(options.trim === true && opts.ltrim !== false){ + options.ltrim = true; + }else if(options.ltrim !== true){ + options.ltrim = false; + } + if(options.trim === true && opts.rtrim !== false){ + options.rtrim = true; + }else if(options.rtrim !== true){ + options.rtrim = false; + } + // Normalize option `to` + if(options.to === undefined || options.to === null){ + options.to = -1; + }else { + if(typeof options.to === 'string' && /\d+/.test(options.to)){ + options.to = parseInt(options.to); + } + if(Number.isInteger(options.to)){ + if(options.to <= 0){ + throw new Error(`Invalid Option: to must be a positive integer greater than 0, got ${JSON.stringify(opts.to)}`); + } }else { - throw new Error(`Invalid Option: relax_quotes must be a boolean, got ${JSON.stringify(options.relax_quotes)}`); + throw new Error(`Invalid Option: to must be an integer, got ${JSON.stringify(opts.to)}`); } - // Normalize option `skip_empty_lines` - if(typeof options.skip_empty_lines === 'boolean');else if(options.skip_empty_lines === undefined || options.skip_empty_lines === null){ - options.skip_empty_lines = false; - }else { - throw new Error(`Invalid Option: skip_empty_lines must be a boolean, got ${JSON.stringify(options.skip_empty_lines)}`); + } + // Normalize option `to_line` + if(options.to_line === undefined || options.to_line === null){ + options.to_line = -1; + }else { + if(typeof options.to_line === 'string' && /\d+/.test(options.to_line)){ + options.to_line = parseInt(options.to_line); } - // Normalize option `skip_records_with_empty_values` - if(typeof options.skip_records_with_empty_values === 'boolean');else if(options.skip_records_with_empty_values === undefined || options.skip_records_with_empty_values === null){ - options.skip_records_with_empty_values = false; + if(Number.isInteger(options.to_line)){ + if(options.to_line <= 0){ + throw new Error(`Invalid Option: to_line must be a positive integer greater than 0, got ${JSON.stringify(opts.to_line)}`); + } }else { - throw new Error(`Invalid Option: skip_records_with_empty_values must be a boolean, got ${JSON.stringify(options.skip_records_with_empty_values)}`); + throw new Error(`Invalid Option: to_line must be an integer, got ${JSON.stringify(opts.to_line)}`); } - // Normalize option `skip_records_with_error` - if(typeof options.skip_records_with_error === 'boolean');else if(options.skip_records_with_error === undefined || options.skip_records_with_error === null){ - options.skip_records_with_error = false; - }else { - throw new Error(`Invalid Option: skip_records_with_error must be a boolean, got ${JSON.stringify(options.skip_records_with_error)}`); - } - // Normalize option `rtrim` - if(options.rtrim === undefined || options.rtrim === null || options.rtrim === false){ - options.rtrim = false; - }else if(options.rtrim !== true){ - throw new Error(`Invalid Option: rtrim must be a boolean, got ${JSON.stringify(options.rtrim)}`); - } - // Normalize option `ltrim` - if(options.ltrim === undefined || options.ltrim === null || options.ltrim === false){ - options.ltrim = false; - }else if(options.ltrim !== true){ - throw new Error(`Invalid Option: ltrim must be a boolean, got ${JSON.stringify(options.ltrim)}`); - } - // Normalize option `trim` - if(options.trim === undefined || options.trim === null || options.trim === false){ - options.trim = false; - }else if(options.trim !== true){ - throw new Error(`Invalid Option: trim must be a boolean, got ${JSON.stringify(options.trim)}`); - } - // Normalize options `trim`, `ltrim` and `rtrim` - if(options.trim === true && opts.ltrim !== false){ - options.ltrim = true; - }else if(options.ltrim !== true){ - options.ltrim = false; - } - if(options.trim === true && opts.rtrim !== false){ - options.rtrim = true; - }else if(options.rtrim !== true){ - options.rtrim = false; - } - // Normalize option `to` - if(options.to === undefined || options.to === null){ - options.to = -1; - }else { - if(typeof options.to === 'string' && /\d+/.test(options.to)){ - options.to = parseInt(options.to); - } - if(Number.isInteger(options.to)){ - if(options.to <= 0){ - throw new Error(`Invalid Option: to must be a positive integer greater than 0, got ${JSON.stringify(opts.to)}`); + } + return options; + }; + + const isRecordEmpty = function(record){ + return record.every((field) => field == null || field.toString && field.toString().trim() === ''); + }; + + // white space characters + // https://en.wikipedia.org/wiki/Whitespace_character + // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions/Character_Classes#Types + // \f\n\r\t\v\u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff + const tab = 9; + const nl = 10; // \n, 0x0A in hexadecimal, 10 in decimal + const np = 12; + const cr = 13; // \r, 0x0D in hexadécimal, 13 in decimal + const space = 32; + const boms = { + // Note, the following are equals: + // Buffer.from("\ufeff") + // Buffer.from([239, 187, 191]) + // Buffer.from('EFBBBF', 'hex') + 'utf8': Buffer.from([239, 187, 191]), + // Note, the following are equals: + // Buffer.from "\ufeff", 'utf16le + // Buffer.from([255, 254]) + 'utf16le': Buffer.from([255, 254]) + }; + + const transform = function(original_options = {}) { + const info = { + bytes: 0, + comment_lines: 0, + empty_lines: 0, + invalid_field_length: 0, + lines: 1, + records: 0 + }; + const options = normalize_options(original_options); + return { + info: info, + original_options: original_options, + options: options, + state: init_state(options), + __needMoreData: function(i, bufLen, end){ + if(end) return false; + const {quote} = this.options; + const {quoting, needMoreDataSize, recordDelimiterMaxLength} = this.state; + const numOfCharLeft = bufLen - i - 1; + const requiredLength = Math.max( + needMoreDataSize, + // Skip if the remaining buffer smaller than record delimiter + recordDelimiterMaxLength, + // Skip if the remaining buffer can be record delimiter following the closing quote + // 1 is for quote.length + quoting ? (quote.length + recordDelimiterMaxLength) : 0, + ); + return numOfCharLeft < requiredLength; + }, + // Central parser implementation + parse: function(nextBuf, end, push, close){ + const {bom, comment, escape, from_line, ltrim, max_record_size, quote, raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options; + let {record_delimiter} = this.options; + const {bomSkipped, previousBuf, rawBuffer, escapeIsQuote} = this.state; + let buf; + if(previousBuf === undefined){ + if(nextBuf === undefined){ + // Handle empty string + close(); + return; + }else { + buf = nextBuf; } + }else if(previousBuf !== undefined && nextBuf === undefined){ + buf = previousBuf; }else { - throw new Error(`Invalid Option: to must be an integer, got ${JSON.stringify(opts.to)}`); + buf = Buffer.concat([previousBuf, nextBuf]); } - } - // Normalize option `to_line` - if(options.to_line === undefined || options.to_line === null){ - options.to_line = -1; - }else { - if(typeof options.to_line === 'string' && /\d+/.test(options.to_line)){ - options.to_line = parseInt(options.to_line); - } - if(Number.isInteger(options.to_line)){ - if(options.to_line <= 0){ - throw new Error(`Invalid Option: to_line must be a positive integer greater than 0, got ${JSON.stringify(opts.to_line)}`); + // Handle UTF BOM + if(bomSkipped === false){ + if(bom === false){ + this.state.bomSkipped = true; + }else if(buf.length < 3){ + // No enough data + if(end === false){ + // Wait for more data + this.state.previousBuf = buf; + return; + } + }else { + for(const encoding in boms){ + if(boms[encoding].compare(buf, 0, boms[encoding].length) === 0){ + // Skip BOM + const bomLength = boms[encoding].length; + this.state.bufBytesStart += bomLength; + buf = buf.slice(bomLength); + // Renormalize original options with the new encoding + this.options = normalize_options({...this.original_options, encoding: encoding}); + break; + } + } + this.state.bomSkipped = true; } - }else { - throw new Error(`Invalid Option: to_line must be an integer, got ${JSON.stringify(opts.to_line)}`); } - } - this.info = { - bytes: 0, - comment_lines: 0, - empty_lines: 0, - invalid_field_length: 0, - lines: 1, - records: 0 - }; - this.options = options; - this.state = { - bomSkipped: false, - bufBytesStart: 0, - castField: fnCastField, - commenting: false, - // Current error encountered by a record - error: undefined, - enabled: options.from_line === 1, - escaping: false, - escapeIsQuote: isBuffer(options.escape) && isBuffer(options.quote) && Buffer.compare(options.escape, options.quote) === 0, - // columns can be `false`, `true`, `Array` - expectedRecordLength: Array.isArray(options.columns) ? options.columns.length : undefined, - field: new ResizeableBuffer(20), - firstLineToHeaders: fnFirstLineToHeaders, - needMoreDataSize: Math.max( - // Skip if the remaining buffer smaller than comment - options.comment !== null ? options.comment.length : 0, - // Skip if the remaining buffer can be delimiter - ...options.delimiter.map((delimiter) => delimiter.length), - // Skip if the remaining buffer can be escape sequence - options.quote !== null ? options.quote.length : 0, - ), - previousBuf: undefined, - quoting: false, - stop: false, - rawBuffer: new ResizeableBuffer(100), - record: [], - recordHasError: false, - record_length: 0, - recordDelimiterMaxLength: options.record_delimiter.length === 0 ? 2 : Math.max(...options.record_delimiter.map((v) => v.length)), - trimChars: [Buffer.from(' ', options.encoding)[0], Buffer.from('\t', options.encoding)[0]], - wasQuoting: false, - wasRowDelimiter: false - }; - } - // Implementation of `Transform._transform` - _transform(buf, encoding, callback){ - if(this.state.stop === true){ - return; - } - const err = this.__parse(buf, false); - if(err !== undefined){ - this.state.stop = true; - } - callback(err); - } - // Implementation of `Transform._flush` - _flush(callback){ - if(this.state.stop === true){ - return; - } - const err = this.__parse(undefined, true); - callback(err); - } - // Central parser implementation - __parse(nextBuf, end){ - const {bom, comment, escape, from_line, ltrim, max_record_size, quote, raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options; - let {record_delimiter} = this.options; - const {bomSkipped, previousBuf, rawBuffer, escapeIsQuote} = this.state; - let buf; - if(previousBuf === undefined){ - if(nextBuf === undefined){ - // Handle empty string - this.push(null); - return; - }else { - buf = nextBuf; - } - }else if(previousBuf !== undefined && nextBuf === undefined){ - buf = previousBuf; - }else { - buf = Buffer.concat([previousBuf, nextBuf]); - } - // Handle UTF BOM - if(bomSkipped === false){ - if(bom === false){ - this.state.bomSkipped = true; - }else if(buf.length < 3){ - // No enough data - if(end === false){ - // Wait for more data - this.state.previousBuf = buf; + const bufLen = buf.length; + let pos; + for(pos = 0; pos < bufLen; pos++){ + // Ensure we get enough space to look ahead + // There should be a way to move this out of the loop + if(this.__needMoreData(pos, bufLen, end)){ + break; + } + if(this.state.wasRowDelimiter === true){ + this.info.lines++; + this.state.wasRowDelimiter = false; + } + if(to_line !== -1 && this.info.lines > to_line){ + this.state.stop = true; + close(); return; } - }else { - for(const encoding in boms){ - if(boms[encoding].compare(buf, 0, boms[encoding].length) === 0){ - // Skip BOM - const bomLength = boms[encoding].length; - this.state.bufBytesStart += bomLength; - buf = buf.slice(bomLength); - // Renormalize original options with the new encoding - this.__normalizeOptions({...this.__originalOptions, encoding: encoding}); - break; + // Auto discovery of record_delimiter, unix, mac and windows supported + if(this.state.quoting === false && record_delimiter.length === 0){ + const record_delimiterCount = this.__autoDiscoverRecordDelimiter(buf, pos); + if(record_delimiterCount){ + record_delimiter = this.options.record_delimiter; } } - this.state.bomSkipped = true; - } - } - const bufLen = buf.length; - let pos; - for(pos = 0; pos < bufLen; pos++){ - // Ensure we get enough space to look ahead - // There should be a way to move this out of the loop - if(this.__needMoreData(pos, bufLen, end)){ - break; - } - if(this.state.wasRowDelimiter === true){ - this.info.lines++; - this.state.wasRowDelimiter = false; - } - if(to_line !== -1 && this.info.lines > to_line){ - this.state.stop = true; - this.push(null); - return; - } - // Auto discovery of record_delimiter, unix, mac and windows supported - if(this.state.quoting === false && record_delimiter.length === 0){ - const record_delimiterCount = this.__autoDiscoverRecordDelimiter(buf, pos); - if(record_delimiterCount){ - record_delimiter = this.options.record_delimiter; + const chr = buf[pos]; + if(raw === true){ + rawBuffer.append(chr); } - } - const chr = buf[pos]; - if(raw === true){ - rawBuffer.append(chr); - } - if((chr === cr || chr === nl) && this.state.wasRowDelimiter === false){ - this.state.wasRowDelimiter = true; - } - // Previous char was a valid escape char - // treat the current char as a regular char - if(this.state.escaping === true){ - this.state.escaping = false; - }else { - // Escape is only active inside quoted fields - // We are quoting, the char is an escape chr and there is a chr to escape - // if(escape !== null && this.state.quoting === true && chr === escape && pos + 1 < bufLen){ - if(escape !== null && this.state.quoting === true && this.__isEscape(buf, pos, chr) && pos + escape.length < bufLen){ - if(escapeIsQuote){ - if(this.__isQuote(buf, pos+escape.length)){ + if((chr === cr || chr === nl) && this.state.wasRowDelimiter === false){ + this.state.wasRowDelimiter = true; + } + // Previous char was a valid escape char + // treat the current char as a regular char + if(this.state.escaping === true){ + this.state.escaping = false; + }else { + // Escape is only active inside quoted fields + // We are quoting, the char is an escape chr and there is a chr to escape + // if(escape !== null && this.state.quoting === true && chr === escape && pos + 1 < bufLen){ + if(escape !== null && this.state.quoting === true && this.__isEscape(buf, pos, chr) && pos + escape.length < bufLen){ + if(escapeIsQuote){ + if(this.__isQuote(buf, pos+escape.length)){ + this.state.escaping = true; + pos += escape.length - 1; + continue; + } + }else { this.state.escaping = true; pos += escape.length - 1; continue; } - }else { - this.state.escaping = true; - pos += escape.length - 1; - continue; } - } - // Not currently escaping and chr is a quote - // TODO: need to compare bytes instead of single char - if(this.state.commenting === false && this.__isQuote(buf, pos)){ - if(this.state.quoting === true){ - const nextChr = buf[pos+quote.length]; - const isNextChrTrimable = rtrim && this.__isCharTrimable(nextChr); - const isNextChrComment = comment !== null && this.__compareBytes(comment, buf, pos+quote.length, nextChr); - const isNextChrDelimiter = this.__isDelimiter(buf, pos+quote.length, nextChr); - const isNextChrRecordDelimiter = record_delimiter.length === 0 ? this.__autoDiscoverRecordDelimiter(buf, pos+quote.length) : this.__isRecordDelimiter(nextChr, buf, pos+quote.length); - // Escape a quote - // Treat next char as a regular character - if(escape !== null && this.__isEscape(buf, pos, chr) && this.__isQuote(buf, pos + escape.length)){ - pos += escape.length - 1; - }else if(!nextChr || isNextChrDelimiter || isNextChrRecordDelimiter || isNextChrComment || isNextChrTrimable){ - this.state.quoting = false; - this.state.wasQuoting = true; - pos += quote.length - 1; - continue; - }else if(relax_quotes === false){ - const err = this.__error( - new CsvError('CSV_INVALID_CLOSING_QUOTE', [ - 'Invalid Closing Quote:', - `got "${String.fromCharCode(nextChr)}"`, - `at line ${this.info.lines}`, - 'instead of delimiter, record delimiter, trimable character', - '(if activated) or comment', - ], this.options, this.__infoField()) - ); - if(err !== undefined) return err; - }else { - this.state.quoting = false; - this.state.wasQuoting = true; - this.state.field.prepend(quote); - pos += quote.length - 1; - } - }else { - if(this.state.field.length !== 0){ - // In relax_quotes mode, treat opening quote preceded by chrs as regular - if(relax_quotes === false){ + // Not currently escaping and chr is a quote + // TODO: need to compare bytes instead of single char + if(this.state.commenting === false && this.__isQuote(buf, pos)){ + if(this.state.quoting === true){ + const nextChr = buf[pos+quote.length]; + const isNextChrTrimable = rtrim && this.__isCharTrimable(nextChr); + const isNextChrComment = comment !== null && this.__compareBytes(comment, buf, pos+quote.length, nextChr); + const isNextChrDelimiter = this.__isDelimiter(buf, pos+quote.length, nextChr); + const isNextChrRecordDelimiter = record_delimiter.length === 0 ? this.__autoDiscoverRecordDelimiter(buf, pos+quote.length) : this.__isRecordDelimiter(nextChr, buf, pos+quote.length); + // Escape a quote + // Treat next char as a regular character + if(escape !== null && this.__isEscape(buf, pos, chr) && this.__isQuote(buf, pos + escape.length)){ + pos += escape.length - 1; + }else if(!nextChr || isNextChrDelimiter || isNextChrRecordDelimiter || isNextChrComment || isNextChrTrimable){ + this.state.quoting = false; + this.state.wasQuoting = true; + pos += quote.length - 1; + continue; + }else if(relax_quotes === false){ const err = this.__error( - new CsvError('INVALID_OPENING_QUOTE', [ - 'Invalid Opening Quote:', - `a quote is found inside a field at line ${this.info.lines}`, - ], this.options, this.__infoField(), { - field: this.state.field, - }) + new CsvError('CSV_INVALID_CLOSING_QUOTE', [ + 'Invalid Closing Quote:', + `got "${String.fromCharCode(nextChr)}"`, + `at line ${this.info.lines}`, + 'instead of delimiter, record delimiter, trimable character', + '(if activated) or comment', + ], this.options, this.__infoField()) ); if(err !== undefined) return err; + }else { + this.state.quoting = false; + this.state.wasQuoting = true; + this.state.field.prepend(quote); + pos += quote.length - 1; } }else { - this.state.quoting = true; - pos += quote.length - 1; - continue; - } - } - } - if(this.state.quoting === false){ - const recordDelimiterLength = this.__isRecordDelimiter(chr, buf, pos); - if(recordDelimiterLength !== 0){ - // Do not emit comments which take a full line - const skipCommentLine = this.state.commenting && (this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0); - if(skipCommentLine){ - this.info.comment_lines++; - // Skip full comment line - }else { - // Activate records emition if above from_line - if(this.state.enabled === false && this.info.lines + (this.state.wasRowDelimiter === true ? 1: 0) >= from_line){ - this.state.enabled = true; - this.__resetField(); - this.__resetRecord(); - pos += recordDelimiterLength - 1; + if(this.state.field.length !== 0){ + // In relax_quotes mode, treat opening quote preceded by chrs as regular + if(relax_quotes === false){ + const err = this.__error( + new CsvError('INVALID_OPENING_QUOTE', [ + 'Invalid Opening Quote:', + `a quote is found inside a field at line ${this.info.lines}`, + ], this.options, this.__infoField(), { + field: this.state.field, + }) + ); + if(err !== undefined) return err; + } + }else { + this.state.quoting = true; + pos += quote.length - 1; continue; } - // Skip if line is empty and skip_empty_lines activated - if(skip_empty_lines === true && this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0){ - this.info.empty_lines++; - pos += recordDelimiterLength - 1; - continue; + } + } + if(this.state.quoting === false){ + const recordDelimiterLength = this.__isRecordDelimiter(chr, buf, pos); + if(recordDelimiterLength !== 0){ + // Do not emit comments which take a full line + const skipCommentLine = this.state.commenting && (this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0); + if(skipCommentLine){ + this.info.comment_lines++; + // Skip full comment line + }else { + // Activate records emition if above from_line + if(this.state.enabled === false && this.info.lines + (this.state.wasRowDelimiter === true ? 1: 0) >= from_line){ + this.state.enabled = true; + this.__resetField(); + this.__resetRecord(); + pos += recordDelimiterLength - 1; + continue; + } + // Skip if line is empty and skip_empty_lines activated + if(skip_empty_lines === true && this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0){ + this.info.empty_lines++; + pos += recordDelimiterLength - 1; + continue; + } + this.info.bytes = this.state.bufBytesStart + pos; + const errField = this.__onField(); + if(errField !== undefined) return errField; + this.info.bytes = this.state.bufBytesStart + pos + recordDelimiterLength; + const errRecord = this.__onRecord(push); + if(errRecord !== undefined) return errRecord; + if(to !== -1 && this.info.records >= to){ + this.state.stop = true; + close(); + return; + } } + this.state.commenting = false; + pos += recordDelimiterLength - 1; + continue; + } + if(this.state.commenting){ + continue; + } + const commentCount = comment === null ? 0 : this.__compareBytes(comment, buf, pos, chr); + if(commentCount !== 0){ + this.state.commenting = true; + continue; + } + const delimiterLength = this.__isDelimiter(buf, pos, chr); + if(delimiterLength !== 0){ this.info.bytes = this.state.bufBytesStart + pos; const errField = this.__onField(); if(errField !== undefined) return errField; - this.info.bytes = this.state.bufBytesStart + pos + recordDelimiterLength; - const errRecord = this.__onRecord(); - if(errRecord !== undefined) return errRecord; - if(to !== -1 && this.info.records >= to){ - this.state.stop = true; - this.push(null); - return; - } + pos += delimiterLength - 1; + continue; } - this.state.commenting = false; - pos += recordDelimiterLength - 1; - continue; - } - if(this.state.commenting){ - continue; } - const commentCount = comment === null ? 0 : this.__compareBytes(comment, buf, pos, chr); - if(commentCount !== 0){ - this.state.commenting = true; - continue; - } - const delimiterLength = this.__isDelimiter(buf, pos, chr); - if(delimiterLength !== 0){ - this.info.bytes = this.state.bufBytesStart + pos; - const errField = this.__onField(); - if(errField !== undefined) return errField; - pos += delimiterLength - 1; - continue; + } + if(this.state.commenting === false){ + if(max_record_size !== 0 && this.state.record_length + this.state.field.length > max_record_size){ + const err = this.__error( + new CsvError('CSV_MAX_RECORD_SIZE', [ + 'Max Record Size:', + 'record exceed the maximum number of tolerated bytes', + `of ${max_record_size}`, + `at line ${this.info.lines}`, + ], this.options, this.__infoField()) + ); + if(err !== undefined) return err; } } - } - if(this.state.commenting === false){ - if(max_record_size !== 0 && this.state.record_length + this.state.field.length > max_record_size){ + const lappend = ltrim === false || this.state.quoting === true || this.state.field.length !== 0 || !this.__isCharTrimable(chr); + // rtrim in non quoting is handle in __onField + const rappend = rtrim === false || this.state.wasQuoting === false; + if(lappend === true && rappend === true){ + this.state.field.append(chr); + }else if(rtrim === true && !this.__isCharTrimable(chr)){ const err = this.__error( - new CsvError('CSV_MAX_RECORD_SIZE', [ - 'Max Record Size:', - 'record exceed the maximum number of tolerated bytes', - `of ${max_record_size}`, + new CsvError('CSV_NON_TRIMABLE_CHAR_AFTER_CLOSING_QUOTE', [ + 'Invalid Closing Quote:', + 'found non trimable byte after quote', `at line ${this.info.lines}`, ], this.options, this.__infoField()) ); if(err !== undefined) return err; } } - const lappend = ltrim === false || this.state.quoting === true || this.state.field.length !== 0 || !this.__isCharTrimable(chr); - // rtrim in non quoting is handle in __onField - const rappend = rtrim === false || this.state.wasQuoting === false; - if(lappend === true && rappend === true){ - this.state.field.append(chr); - }else if(rtrim === true && !this.__isCharTrimable(chr)){ - const err = this.__error( - new CsvError('CSV_NON_TRIMABLE_CHAR_AFTER_CLOSING_QUOTE', [ - 'Invalid Closing Quote:', - 'found non trimable byte after quote', - `at line ${this.info.lines}`, - ], this.options, this.__infoField()) - ); - if(err !== undefined) return err; - } - } - if(end === true){ - // Ensure we are not ending in a quoting state - if(this.state.quoting === true){ - const err = this.__error( - new CsvError('CSV_QUOTE_NOT_CLOSED', [ - 'Quote Not Closed:', - `the parsing is finished with an opening quote at line ${this.info.lines}`, - ], this.options, this.__infoField()) - ); - if(err !== undefined) return err; + if(end === true){ + // Ensure we are not ending in a quoting state + if(this.state.quoting === true){ + const err = this.__error( + new CsvError('CSV_QUOTE_NOT_CLOSED', [ + 'Quote Not Closed:', + `the parsing is finished with an opening quote at line ${this.info.lines}`, + ], this.options, this.__infoField()) + ); + if(err !== undefined) return err; + }else { + // Skip last line if it has no characters + if(this.state.wasQuoting === true || this.state.record.length !== 0 || this.state.field.length !== 0){ + this.info.bytes = this.state.bufBytesStart + pos; + const errField = this.__onField(); + if(errField !== undefined) return errField; + const errRecord = this.__onRecord(push); + if(errRecord !== undefined) return errRecord; + }else if(this.state.wasRowDelimiter === true){ + this.info.empty_lines++; + }else if(this.state.commenting === true){ + this.info.comment_lines++; + } + } }else { - // Skip last line if it has no characters - if(this.state.wasQuoting === true || this.state.record.length !== 0 || this.state.field.length !== 0){ - this.info.bytes = this.state.bufBytesStart + pos; - const errField = this.__onField(); - if(errField !== undefined) return errField; - const errRecord = this.__onRecord(); - if(errRecord !== undefined) return errRecord; - }else if(this.state.wasRowDelimiter === true){ - this.info.empty_lines++; - }else if(this.state.commenting === true){ - this.info.comment_lines++; + this.state.bufBytesStart += pos; + this.state.previousBuf = buf.slice(pos); + } + if(this.state.wasRowDelimiter === true){ + this.info.lines++; + this.state.wasRowDelimiter = false; + } + }, + __onRecord: function(push){ + const {columns, group_columns_by_name, encoding, info, from, relax_column_count, relax_column_count_less, relax_column_count_more, raw, skip_records_with_empty_values} = this.options; + const {enabled, record} = this.state; + if(enabled === false){ + return this.__resetRecord(); + } + // Convert the first line into column names + const recordLength = record.length; + if(columns === true){ + if(skip_records_with_empty_values === true && isRecordEmpty(record)){ + this.__resetRecord(); + return; + } + return this.__firstLineToColumns(record); + } + if(columns === false && this.info.records === 0){ + this.state.expectedRecordLength = recordLength; + } + if(recordLength !== this.state.expectedRecordLength){ + const err = columns === false ? + new CsvError('CSV_RECORD_INCONSISTENT_FIELDS_LENGTH', [ + 'Invalid Record Length:', + `expect ${this.state.expectedRecordLength},`, + `got ${recordLength} on line ${this.info.lines}`, + ], this.options, this.__infoField(), { + record: record, + }) + : + new CsvError('CSV_RECORD_INCONSISTENT_COLUMNS', [ + 'Invalid Record Length:', + `columns length is ${columns.length},`, // rename columns + `got ${recordLength} on line ${this.info.lines}`, + ], this.options, this.__infoField(), { + record: record, + }); + if(relax_column_count === true || + (relax_column_count_less === true && recordLength < this.state.expectedRecordLength) || + (relax_column_count_more === true && recordLength > this.state.expectedRecordLength)){ + this.info.invalid_field_length++; + this.state.error = err; + // Error is undefined with skip_records_with_error + }else { + const finalErr = this.__error(err); + if(finalErr) return finalErr; } } - }else { - this.state.bufBytesStart += pos; - this.state.previousBuf = buf.slice(pos); - } - if(this.state.wasRowDelimiter === true){ - this.info.lines++; - this.state.wasRowDelimiter = false; - } - } - __onRecord(){ - const {columns, group_columns_by_name, encoding, info, from, relax_column_count, relax_column_count_less, relax_column_count_more, raw, skip_records_with_empty_values} = this.options; - const {enabled, record} = this.state; - if(enabled === false){ - return this.__resetRecord(); - } - // Convert the first line into column names - const recordLength = record.length; - if(columns === true){ if(skip_records_with_empty_values === true && isRecordEmpty(record)){ this.__resetRecord(); return; } - return this.__firstLineToColumns(record); - } - if(columns === false && this.info.records === 0){ - this.state.expectedRecordLength = recordLength; - } - if(recordLength !== this.state.expectedRecordLength){ - const err = columns === false ? - new CsvError('CSV_RECORD_INCONSISTENT_FIELDS_LENGTH', [ - 'Invalid Record Length:', - `expect ${this.state.expectedRecordLength},`, - `got ${recordLength} on line ${this.info.lines}`, - ], this.options, this.__infoField(), { - record: record, - }) - : - new CsvError('CSV_RECORD_INCONSISTENT_COLUMNS', [ - 'Invalid Record Length:', - `columns length is ${columns.length},`, // rename columns - `got ${recordLength} on line ${this.info.lines}`, - ], this.options, this.__infoField(), { - record: record, - }); - if(relax_column_count === true || - (relax_column_count_less === true && recordLength < this.state.expectedRecordLength) || - (relax_column_count_more === true && recordLength > this.state.expectedRecordLength)){ - this.info.invalid_field_length++; - this.state.error = err; - // Error is undefined with skip_records_with_error - }else { - const finalErr = this.__error(err); - if(finalErr) return finalErr; + if(this.state.recordHasError === true){ + this.__resetRecord(); + this.state.recordHasError = false; + return; } - } - if(skip_records_with_empty_values === true && isRecordEmpty(record)){ - this.__resetRecord(); - return; - } - if(this.state.recordHasError === true){ - this.__resetRecord(); - this.state.recordHasError = false; - return; - } - this.info.records++; - if(from === 1 || this.info.records >= from){ - const {objname} = this.options; - // With columns, records are object - if(columns !== false){ - const obj = {}; - // Transform record array to an object - for(let i = 0, l = record.length; i < l; i++){ - if(columns[i] === undefined || columns[i].disabled) continue; - // Turn duplicate columns into an array - if (group_columns_by_name === true && obj[columns[i].name] !== undefined) { - if (Array.isArray(obj[columns[i].name])) { - obj[columns[i].name] = obj[columns[i].name].concat(record[i]); + this.info.records++; + if(from === 1 || this.info.records >= from){ + const {objname} = this.options; + // With columns, records are object + if(columns !== false){ + const obj = {}; + // Transform record array to an object + for(let i = 0, l = record.length; i < l; i++){ + if(columns[i] === undefined || columns[i].disabled) continue; + // Turn duplicate columns into an array + if (group_columns_by_name === true && obj[columns[i].name] !== undefined) { + if (Array.isArray(obj[columns[i].name])) { + obj[columns[i].name] = obj[columns[i].name].concat(record[i]); + } else { + obj[columns[i].name] = [obj[columns[i].name], record[i]]; + } } else { - obj[columns[i].name] = [obj[columns[i].name], record[i]]; + obj[columns[i].name] = record[i]; } - } else { - obj[columns[i].name] = record[i]; } - } - // Without objname (default) - if(raw === true || info === true){ - const extRecord = Object.assign( - {record: obj}, - (raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}), - (info === true ? {info: this.__infoRecord()}: {}) - ); - const err = this.__push( - objname === undefined ? extRecord : [obj[objname], extRecord] - ); - if(err){ - return err; - } - }else { - const err = this.__push( - objname === undefined ? obj : [obj[objname], obj] - ); - if(err){ - return err; - } - } - // Without columns, records are array - }else { - if(raw === true || info === true){ - const extRecord = Object.assign( - {record: record}, - raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}, - info === true ? {info: this.__infoRecord()}: {} - ); - const err = this.__push( - objname === undefined ? extRecord : [record[objname], extRecord] - ); - if(err){ - return err; + // Without objname (default) + if(raw === true || info === true){ + const extRecord = Object.assign( + {record: obj}, + (raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}), + (info === true ? {info: this.__infoRecord()}: {}) + ); + const err = this.__push( + objname === undefined ? extRecord : [obj[objname], extRecord] + , push); + if(err){ + return err; + } + }else { + const err = this.__push( + objname === undefined ? obj : [obj[objname], obj] + , push); + if(err){ + return err; + } } + // Without columns, records are array }else { - const err = this.__push( - objname === undefined ? record : [record[objname], record] - ); - if(err){ - return err; + if(raw === true || info === true){ + const extRecord = Object.assign( + {record: record}, + raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}, + info === true ? {info: this.__infoRecord()}: {} + ); + const err = this.__push( + objname === undefined ? extRecord : [record[objname], extRecord] + , push); + if(err){ + return err; + } + }else { + const err = this.__push( + objname === undefined ? record : [record[objname], record] + , push); + if(err){ + return err; + } } } } - } - this.__resetRecord(); - } - __firstLineToColumns(record){ - const {firstLineToHeaders} = this.state; - try{ - const headers = firstLineToHeaders === undefined ? record : firstLineToHeaders.call(null, record); - if(!Array.isArray(headers)){ - return this.__error( - new CsvError('CSV_INVALID_COLUMN_MAPPING', [ - 'Invalid Column Mapping:', - 'expect an array from column function,', - `got ${JSON.stringify(headers)}` - ], this.options, this.__infoField(), { - headers: headers, - }) - ); - } - const normalizedHeaders = normalizeColumnsArray(headers); - this.state.expectedRecordLength = normalizedHeaders.length; - this.options.columns = normalizedHeaders; this.__resetRecord(); - return; - }catch(err){ - return err; - } - } - __resetRecord(){ - if(this.options.raw === true){ - this.state.rawBuffer.reset(); - } - this.state.error = undefined; - this.state.record = []; - this.state.record_length = 0; - } - __onField(){ - const {cast, encoding, rtrim, max_record_size} = this.options; - const {enabled, wasQuoting} = this.state; - // Short circuit for the from_line options - if(enabled === false){ - return this.__resetField(); - } - let field = this.state.field.toString(encoding); - if(rtrim === true && wasQuoting === false){ - field = field.trimRight(); - } - if(cast === true){ - const [err, f] = this.__cast(field); - if(err !== undefined) return err; - field = f; - } - this.state.record.push(field); - // Increment record length if record size must not exceed a limit - if(max_record_size !== 0 && typeof field === 'string'){ - this.state.record_length += field.length; - } - this.__resetField(); - } - __resetField(){ - this.state.field.reset(); - this.state.wasQuoting = false; - } - __push(record){ - const {on_record} = this.options; - if(on_record !== undefined){ - const info = this.__infoRecord(); + }, + __firstLineToColumns: function(record){ + const {firstLineToHeaders} = this.state; try{ - record = on_record.call(null, record, info); + const headers = firstLineToHeaders === undefined ? record : firstLineToHeaders.call(null, record); + if(!Array.isArray(headers)){ + return this.__error( + new CsvError('CSV_INVALID_COLUMN_MAPPING', [ + 'Invalid Column Mapping:', + 'expect an array from column function,', + `got ${JSON.stringify(headers)}` + ], this.options, this.__infoField(), { + headers: headers, + }) + ); + } + const normalizedHeaders = normalize_columns_array(headers); + this.state.expectedRecordLength = normalizedHeaders.length; + this.options.columns = normalizedHeaders; + this.__resetRecord(); + return; }catch(err){ return err; } - if(record === undefined || record === null){ return; } - } - this.push(record); - } - // Return a tuple with the error and the casted value - __cast(field){ - const {columns, relax_column_count} = this.options; - const isColumns = Array.isArray(columns); - // Dont loose time calling cast - // because the final record is an object - // and this field can't be associated to a key present in columns - if(isColumns === true && relax_column_count && this.options.columns.length <= this.state.record.length){ - return [undefined, undefined]; - } - if(this.state.castField !== null){ - try{ + }, + __resetRecord: function(){ + if(this.options.raw === true){ + this.state.rawBuffer.reset(); + } + this.state.error = undefined; + this.state.record = []; + this.state.record_length = 0; + }, + __onField: function(){ + const {cast, encoding, rtrim, max_record_size} = this.options; + const {enabled, wasQuoting} = this.state; + // Short circuit for the from_line options + if(enabled === false){ + return this.__resetField(); + } + let field = this.state.field.toString(encoding); + if(rtrim === true && wasQuoting === false){ + field = field.trimRight(); + } + if(cast === true){ + const [err, f] = this.__cast(field); + if(err !== undefined) return err; + field = f; + } + this.state.record.push(field); + // Increment record length if record size must not exceed a limit + if(max_record_size !== 0 && typeof field === 'string'){ + this.state.record_length += field.length; + } + this.__resetField(); + }, + __resetField: function(){ + this.state.field.reset(); + this.state.wasQuoting = false; + }, + __push: function(record, push){ + const {on_record} = this.options; + if(on_record !== undefined){ + const info = this.__infoRecord(); + try{ + record = on_record.call(null, record, info); + }catch(err){ + return err; + } + if(record === undefined || record === null){ return; } + } + push(record); + }, + // Return a tuple with the error and the casted value + __cast: function(field){ + const {columns, relax_column_count} = this.options; + const isColumns = Array.isArray(columns); + // Dont loose time calling cast + // because the final record is an object + // and this field can't be associated to a key present in columns + if(isColumns === true && relax_column_count && this.options.columns.length <= this.state.record.length){ + return [undefined, undefined]; + } + if(this.state.castField !== null){ + try{ + const info = this.__infoField(); + return [undefined, this.state.castField.call(null, field, info)]; + }catch(err){ + return [err]; + } + } + if(this.__isFloat(field)){ + return [undefined, parseFloat(field)]; + }else if(this.options.cast_date !== false){ const info = this.__infoField(); - return [undefined, this.state.castField.call(null, field, info)]; - }catch(err){ - return [err]; + return [undefined, this.options.cast_date.call(null, field, info)]; } - } - if(this.__isFloat(field)){ - return [undefined, parseFloat(field)]; - }else if(this.options.cast_date !== false){ - const info = this.__infoField(); - return [undefined, this.options.cast_date.call(null, field, info)]; - } - return [undefined, field]; - } - // Helper to test if a character is a space or a line delimiter - __isCharTrimable(chr){ - return chr === space || chr === tab || chr === cr || chr === nl || chr === np; - } - // Keep it in case we implement the `cast_int` option - // __isInt(value){ - // // return Number.isInteger(parseInt(value)) - // // return !isNaN( parseInt( obj ) ); - // return /^(\-|\+)?[1-9][0-9]*$/.test(value) - // } - __isFloat(value){ - return (value - parseFloat(value) + 1) >= 0; // Borrowed from jquery - } - __compareBytes(sourceBuf, targetBuf, targetPos, firstByte){ - if(sourceBuf[0] !== firstByte) return 0; - const sourceLength = sourceBuf.length; - for(let i = 1; i < sourceLength; i++){ - if(sourceBuf[i] !== targetBuf[targetPos+i]) return 0; - } - return sourceLength; - } - __needMoreData(i, bufLen, end){ - if(end) return false; - const {quote} = this.options; - const {quoting, needMoreDataSize, recordDelimiterMaxLength} = this.state; - const numOfCharLeft = bufLen - i - 1; - const requiredLength = Math.max( - needMoreDataSize, - // Skip if the remaining buffer smaller than record delimiter - recordDelimiterMaxLength, - // Skip if the remaining buffer can be record delimiter following the closing quote - // 1 is for quote.length - quoting ? (quote.length + recordDelimiterMaxLength) : 0, - ); - return numOfCharLeft < requiredLength; - } - __isDelimiter(buf, pos, chr){ - const {delimiter, ignore_last_delimiters} = this.options; - if(ignore_last_delimiters === true && this.state.record.length === this.options.columns.length - 1){ - return 0; - }else if(ignore_last_delimiters !== false && typeof ignore_last_delimiters === 'number' && this.state.record.length === ignore_last_delimiters - 1){ - return 0; - } - loop1: for(let i = 0; i < delimiter.length; i++){ - const del = delimiter[i]; - if(del[0] === chr){ - for(let j = 1; j < del.length; j++){ - if(del[j] !== buf[pos+j]) continue loop1; + return [undefined, field]; + }, + // Helper to test if a character is a space or a line delimiter + __isCharTrimable: function(chr){ + return chr === space || chr === tab || chr === cr || chr === nl || chr === np; + }, + // Keep it in case we implement the `cast_int` option + // __isInt(value){ + // // return Number.isInteger(parseInt(value)) + // // return !isNaN( parseInt( obj ) ); + // return /^(\-|\+)?[1-9][0-9]*$/.test(value) + // } + __isFloat: function(value){ + return (value - parseFloat(value) + 1) >= 0; // Borrowed from jquery + }, + __compareBytes: function(sourceBuf, targetBuf, targetPos, firstByte){ + if(sourceBuf[0] !== firstByte) return 0; + const sourceLength = sourceBuf.length; + for(let i = 1; i < sourceLength; i++){ + if(sourceBuf[i] !== targetBuf[targetPos+i]) return 0; + } + return sourceLength; + }, + __isDelimiter: function(buf, pos, chr){ + const {delimiter, ignore_last_delimiters} = this.options; + if(ignore_last_delimiters === true && this.state.record.length === this.options.columns.length - 1){ + return 0; + }else if(ignore_last_delimiters !== false && typeof ignore_last_delimiters === 'number' && this.state.record.length === ignore_last_delimiters - 1){ + return 0; + } + loop1: for(let i = 0; i < delimiter.length; i++){ + const del = delimiter[i]; + if(del[0] === chr){ + for(let j = 1; j < del.length; j++){ + if(del[j] !== buf[pos+j]) continue loop1; + } + return del.length; } - return del.length; } - } - return 0; - } - __isRecordDelimiter(chr, buf, pos){ - const {record_delimiter} = this.options; - const recordDelimiterLength = record_delimiter.length; - loop1: for(let i = 0; i < recordDelimiterLength; i++){ - const rd = record_delimiter[i]; - const rdLength = rd.length; - if(rd[0] !== chr){ - continue; + return 0; + }, + __isRecordDelimiter: function(chr, buf, pos){ + const {record_delimiter} = this.options; + const recordDelimiterLength = record_delimiter.length; + loop1: for(let i = 0; i < recordDelimiterLength; i++){ + const rd = record_delimiter[i]; + const rdLength = rd.length; + if(rd[0] !== chr){ + continue; + } + for(let j = 1; j < rdLength; j++){ + if(rd[j] !== buf[pos+j]){ + continue loop1; + } + } + return rd.length; } - for(let j = 1; j < rdLength; j++){ - if(rd[j] !== buf[pos+j]){ - continue loop1; + return 0; + }, + __isEscape: function(buf, pos, chr){ + const {escape} = this.options; + if(escape === null) return false; + const l = escape.length; + if(escape[0] === chr){ + for(let i = 0; i < l; i++){ + if(escape[i] !== buf[pos+i]){ + return false; + } } + return true; } - return rd.length; - } - return 0; - } - __isEscape(buf, pos, chr){ - const {escape} = this.options; - if(escape === null) return false; - const l = escape.length; - if(escape[0] === chr){ + return false; + }, + __isQuote: function(buf, pos){ + const {quote} = this.options; + if(quote === null) return false; + const l = quote.length; for(let i = 0; i < l; i++){ - if(escape[i] !== buf[pos+i]){ + if(quote[i] !== buf[pos+i]){ return false; } } return true; - } - return false; - } - __isQuote(buf, pos){ - const {quote} = this.options; - if(quote === null) return false; - const l = quote.length; - for(let i = 0; i < l; i++){ - if(quote[i] !== buf[pos+i]){ - return false; - } - } - return true; - } - __autoDiscoverRecordDelimiter(buf, pos){ - const {encoding} = this.options; - const chr = buf[pos]; - if(chr === cr){ - if(buf[pos+1] === nl){ - this.options.record_delimiter.push(Buffer.from('\r\n', encoding)); - this.state.recordDelimiterMaxLength = 2; - return 2; - }else { - this.options.record_delimiter.push(Buffer.from('\r', encoding)); + }, + __autoDiscoverRecordDelimiter: function(buf, pos){ + const {encoding} = this.options; + const chr = buf[pos]; + if(chr === cr){ + if(buf[pos+1] === nl){ + this.options.record_delimiter.push(Buffer.from('\r\n', encoding)); + this.state.recordDelimiterMaxLength = 2; + return 2; + }else { + this.options.record_delimiter.push(Buffer.from('\r', encoding)); + this.state.recordDelimiterMaxLength = 1; + return 1; + } + }else if(chr === nl){ + this.options.record_delimiter.push(Buffer.from('\n', encoding)); this.state.recordDelimiterMaxLength = 1; return 1; } - }else if(chr === nl){ - this.options.record_delimiter.push(Buffer.from('\n', encoding)); - this.state.recordDelimiterMaxLength = 1; - return 1; - } - return 0; - } - __error(msg){ - const {encoding, raw, skip_records_with_error} = this.options; - const err = typeof msg === 'string' ? new Error(msg) : msg; - if(skip_records_with_error){ - this.state.recordHasError = true; - this.emit('skip', err, raw ? this.state.rawBuffer.toString(encoding) : undefined); - return undefined; - }else { - return err; + return 0; + }, + __error: function(msg){ + const {encoding, raw, skip_records_with_error} = this.options; + const err = typeof msg === 'string' ? new Error(msg) : msg; + if(skip_records_with_error){ + this.state.recordHasError = true; + if(this.options.on_skip !== undefined){ + this.options.on_skip(err, raw ? this.state.rawBuffer.toString(encoding) : undefined); + } + // this.emit('skip', err, raw ? this.state.rawBuffer.toString(encoding) : undefined); + return undefined; + }else { + return err; + } + }, + __infoDataSet: function(){ + return { + ...this.info, + columns: this.options.columns + }; + }, + __infoRecord: function(){ + const {columns, raw, encoding} = this.options; + return { + ...this.__infoDataSet(), + error: this.state.error, + header: columns === true, + index: this.state.record.length, + raw: raw ? this.state.rawBuffer.toString(encoding) : undefined + }; + }, + __infoField: function(){ + const {columns} = this.options; + const isColumns = Array.isArray(columns); + return { + ...this.__infoRecord(), + column: isColumns === true ? + (columns.length > this.state.record.length ? + columns[this.state.record.length].name : + null + ) : + this.state.record.length, + quoting: this.state.wasQuoting, + }; } - } - __infoDataSet(){ - return { - ...this.info, - columns: this.options.columns + }; + }; + + class Parser extends Transform { + constructor(opts = {}){ + super({...{readableObjectMode: true}, ...opts, encoding: null}); + this.api = transform(opts); + this.api.options.on_skip = (err, chunk) => { + this.emit('skip', err, chunk); }; + // Backward compatibility + this.state = this.api.state; + this.options = this.api.options; + this.info = this.api.info; } - __infoRecord(){ - const {columns, raw, encoding} = this.options; - return { - ...this.__infoDataSet(), - error: this.state.error, - header: columns === true, - index: this.state.record.length, - raw: raw ? this.state.rawBuffer.toString(encoding) : undefined - }; + // Implementation of `Transform._transform` + _transform(buf, encoding, callback){ + if(this.state.stop === true){ + return; + } + const err = this.api.parse(buf, false, (record) => { + this.push.call(this, record); + }, () => { + this.push.call(this, null); + }); + if(err !== undefined){ + this.state.stop = true; + } + callback(err); } - __infoField(){ - const {columns} = this.options; - const isColumns = Array.isArray(columns); - return { - ...this.__infoRecord(), - column: isColumns === true ? - (columns.length > this.state.record.length ? - columns[this.state.record.length].name : - null - ) : - this.state.record.length, - quoting: this.state.wasQuoting, - }; + // Implementation of `Transform._flush` + _flush(callback){ + if(this.state.stop === true){ + return; + } + const err = this.api.parse(undefined, true, (record) => { + this.push.call(this, record); + }, () => { + this.push.call(this, null); + }); + callback(err); } } @@ -6248,7 +6279,7 @@ const type = typeof argument; if(data === undefined && (typeof argument === 'string' || isBuffer(argument))){ data = argument; - }else if(options === undefined && isObject(argument)){ + }else if(options === undefined && is_object(argument)){ options = argument; }else if(callback === undefined && type === 'function'){ callback = argument; @@ -6273,10 +6304,10 @@ } }); parser.on('error', function(err){ - callback(err, undefined, parser.__infoDataSet()); + callback(err, undefined, parser.api.__infoDataSet()); }); parser.on('end', function(){ - callback(undefined, records, parser.__infoDataSet()); + callback(undefined, records, parser.api.__infoDataSet()); }); } if(data !== undefined){ diff --git a/packages/csv-parse/dist/umd/sync.js b/packages/csv-parse/dist/umd/sync.js index 612fd2870..28133103c 100644 --- a/packages/csv-parse/dist/umd/sync.js +++ b/packages/csv-parse/dist/umd/sync.js @@ -205,7 +205,7 @@ var toString = {}.toString; - var isArray$1 = Array.isArray || function (arr) { + var isArray = Array.isArray || function (arr) { return toString.call(arr) == '[object Array]'; }; @@ -473,7 +473,7 @@ return fromArrayLike(that, obj) } - if (obj.type === 'Buffer' && isArray$1(obj.data)) { + if (obj.type === 'Buffer' && isArray(obj.data)) { return fromArrayLike(that, obj.data) } } @@ -538,7 +538,7 @@ }; Buffer.concat = function concat (list, length) { - if (!isArray$1(list)) { + if (!isArray(list)) { throw new TypeError('"list" argument must be an Array of Buffers') } @@ -1969,3007 +1969,53 @@ return typeof obj.readFloatLE === 'function' && typeof obj.slice === 'function' && isFastBuffer(obj.slice(0, 0)) } - var domain; - - // This constructor is used to store event handlers. Instantiating this is - // faster than explicitly calling `Object.create(null)` to get a "clean" empty - // object (tested with v8 v4.9). - function EventHandlers() {} - EventHandlers.prototype = Object.create(null); - - function EventEmitter() { - EventEmitter.init.call(this); - } - - // nodejs oddity - // require('events') === require('events').EventEmitter - EventEmitter.EventEmitter = EventEmitter; - - EventEmitter.usingDomains = false; - - EventEmitter.prototype.domain = undefined; - EventEmitter.prototype._events = undefined; - EventEmitter.prototype._maxListeners = undefined; - - // By default EventEmitters will print a warning if more than 10 listeners are - // added to it. This is a useful default which helps finding memory leaks. - EventEmitter.defaultMaxListeners = 10; - - EventEmitter.init = function() { - this.domain = null; - if (EventEmitter.usingDomains) { - // if there is an active domain, then attach to it. - if (domain.active ) ; - } - - if (!this._events || this._events === Object.getPrototypeOf(this)._events) { - this._events = new EventHandlers(); - this._eventsCount = 0; - } - - this._maxListeners = this._maxListeners || undefined; - }; - - // Obviously not all Emitters should be limited to 10. This function allows - // that to be increased. Set to zero for unlimited. - EventEmitter.prototype.setMaxListeners = function setMaxListeners(n) { - if (typeof n !== 'number' || n < 0 || isNaN(n)) - throw new TypeError('"n" argument must be a positive number'); - this._maxListeners = n; - return this; - }; - - function $getMaxListeners(that) { - if (that._maxListeners === undefined) - return EventEmitter.defaultMaxListeners; - return that._maxListeners; - } - - EventEmitter.prototype.getMaxListeners = function getMaxListeners() { - return $getMaxListeners(this); - }; - - // These standalone emit* functions are used to optimize calling of event - // handlers for fast cases because emit() itself often has a variable number of - // arguments and can be deoptimized because of that. These functions always have - // the same number of arguments and thus do not get deoptimized, so the code - // inside them can execute faster. - function emitNone(handler, isFn, self) { - if (isFn) - handler.call(self); - else { - var len = handler.length; - var listeners = arrayClone(handler, len); - for (var i = 0; i < len; ++i) - listeners[i].call(self); - } - } - function emitOne(handler, isFn, self, arg1) { - if (isFn) - handler.call(self, arg1); - else { - var len = handler.length; - var listeners = arrayClone(handler, len); - for (var i = 0; i < len; ++i) - listeners[i].call(self, arg1); - } - } - function emitTwo(handler, isFn, self, arg1, arg2) { - if (isFn) - handler.call(self, arg1, arg2); - else { - var len = handler.length; - var listeners = arrayClone(handler, len); - for (var i = 0; i < len; ++i) - listeners[i].call(self, arg1, arg2); - } - } - function emitThree(handler, isFn, self, arg1, arg2, arg3) { - if (isFn) - handler.call(self, arg1, arg2, arg3); - else { - var len = handler.length; - var listeners = arrayClone(handler, len); - for (var i = 0; i < len; ++i) - listeners[i].call(self, arg1, arg2, arg3); - } - } - - function emitMany(handler, isFn, self, args) { - if (isFn) - handler.apply(self, args); - else { - var len = handler.length; - var listeners = arrayClone(handler, len); - for (var i = 0; i < len; ++i) - listeners[i].apply(self, args); - } - } - - EventEmitter.prototype.emit = function emit(type) { - var er, handler, len, args, i, events, domain; - var doError = (type === 'error'); - - events = this._events; - if (events) - doError = (doError && events.error == null); - else if (!doError) - return false; - - domain = this.domain; - - // If there is no 'error' event listener then throw. - if (doError) { - er = arguments[1]; - if (domain) { - if (!er) - er = new Error('Uncaught, unspecified "error" event'); - er.domainEmitter = this; - er.domain = domain; - er.domainThrown = false; - domain.emit('error', er); - } else if (er instanceof Error) { - throw er; // Unhandled 'error' event - } else { - // At least give some kind of context to the user - var err = new Error('Uncaught, unspecified "error" event. (' + er + ')'); - err.context = er; - throw err; - } - return false; - } - - handler = events[type]; - - if (!handler) - return false; - - var isFn = typeof handler === 'function'; - len = arguments.length; - switch (len) { - // fast cases - case 1: - emitNone(handler, isFn, this); - break; - case 2: - emitOne(handler, isFn, this, arguments[1]); - break; - case 3: - emitTwo(handler, isFn, this, arguments[1], arguments[2]); - break; - case 4: - emitThree(handler, isFn, this, arguments[1], arguments[2], arguments[3]); - break; - // slower - default: - args = new Array(len - 1); - for (i = 1; i < len; i++) - args[i - 1] = arguments[i]; - emitMany(handler, isFn, this, args); - } - - return true; - }; - - function _addListener(target, type, listener, prepend) { - var m; - var events; - var existing; - - if (typeof listener !== 'function') - throw new TypeError('"listener" argument must be a function'); - - events = target._events; - if (!events) { - events = target._events = new EventHandlers(); - target._eventsCount = 0; - } else { - // To avoid recursion in the case that type === "newListener"! Before - // adding it to the listeners, first emit "newListener". - if (events.newListener) { - target.emit('newListener', type, - listener.listener ? listener.listener : listener); - - // Re-assign `events` because a newListener handler could have caused the - // this._events to be assigned to a new object - events = target._events; - } - existing = events[type]; - } - - if (!existing) { - // Optimize the case of one listener. Don't need the extra array object. - existing = events[type] = listener; - ++target._eventsCount; - } else { - if (typeof existing === 'function') { - // Adding the second element, need to change to array. - existing = events[type] = prepend ? [listener, existing] : - [existing, listener]; - } else { - // If we've already got an array, just append. - if (prepend) { - existing.unshift(listener); - } else { - existing.push(listener); - } - } - - // Check for listener leak - if (!existing.warned) { - m = $getMaxListeners(target); - if (m && m > 0 && existing.length > m) { - existing.warned = true; - var w = new Error('Possible EventEmitter memory leak detected. ' + - existing.length + ' ' + type + ' listeners added. ' + - 'Use emitter.setMaxListeners() to increase limit'); - w.name = 'MaxListenersExceededWarning'; - w.emitter = target; - w.type = type; - w.count = existing.length; - emitWarning(w); - } - } - } - - return target; - } - function emitWarning(e) { - typeof console.warn === 'function' ? console.warn(e) : console.log(e); - } - EventEmitter.prototype.addListener = function addListener(type, listener) { - return _addListener(this, type, listener, false); - }; - - EventEmitter.prototype.on = EventEmitter.prototype.addListener; - - EventEmitter.prototype.prependListener = - function prependListener(type, listener) { - return _addListener(this, type, listener, true); - }; - - function _onceWrap(target, type, listener) { - var fired = false; - function g() { - target.removeListener(type, g); - if (!fired) { - fired = true; - listener.apply(target, arguments); - } - } - g.listener = listener; - return g; - } - - EventEmitter.prototype.once = function once(type, listener) { - if (typeof listener !== 'function') - throw new TypeError('"listener" argument must be a function'); - this.on(type, _onceWrap(this, type, listener)); - return this; - }; - - EventEmitter.prototype.prependOnceListener = - function prependOnceListener(type, listener) { - if (typeof listener !== 'function') - throw new TypeError('"listener" argument must be a function'); - this.prependListener(type, _onceWrap(this, type, listener)); - return this; - }; - - // emits a 'removeListener' event iff the listener was removed - EventEmitter.prototype.removeListener = - function removeListener(type, listener) { - var list, events, position, i, originalListener; - - if (typeof listener !== 'function') - throw new TypeError('"listener" argument must be a function'); - - events = this._events; - if (!events) - return this; - - list = events[type]; - if (!list) - return this; - - if (list === listener || (list.listener && list.listener === listener)) { - if (--this._eventsCount === 0) - this._events = new EventHandlers(); - else { - delete events[type]; - if (events.removeListener) - this.emit('removeListener', type, list.listener || listener); - } - } else if (typeof list !== 'function') { - position = -1; - - for (i = list.length; i-- > 0;) { - if (list[i] === listener || - (list[i].listener && list[i].listener === listener)) { - originalListener = list[i].listener; - position = i; - break; - } - } - - if (position < 0) - return this; - - if (list.length === 1) { - list[0] = undefined; - if (--this._eventsCount === 0) { - this._events = new EventHandlers(); - return this; - } else { - delete events[type]; - } - } else { - spliceOne(list, position); - } - - if (events.removeListener) - this.emit('removeListener', type, originalListener || listener); - } - - return this; - }; - - EventEmitter.prototype.removeAllListeners = - function removeAllListeners(type) { - var listeners, events; - - events = this._events; - if (!events) - return this; - - // not listening for removeListener, no need to emit - if (!events.removeListener) { - if (arguments.length === 0) { - this._events = new EventHandlers(); - this._eventsCount = 0; - } else if (events[type]) { - if (--this._eventsCount === 0) - this._events = new EventHandlers(); - else - delete events[type]; - } - return this; - } - - // emit removeListener for all listeners on all events - if (arguments.length === 0) { - var keys = Object.keys(events); - for (var i = 0, key; i < keys.length; ++i) { - key = keys[i]; - if (key === 'removeListener') continue; - this.removeAllListeners(key); - } - this.removeAllListeners('removeListener'); - this._events = new EventHandlers(); - this._eventsCount = 0; - return this; - } - - listeners = events[type]; - - if (typeof listeners === 'function') { - this.removeListener(type, listeners); - } else if (listeners) { - // LIFO order - do { - this.removeListener(type, listeners[listeners.length - 1]); - } while (listeners[0]); - } - - return this; - }; - - EventEmitter.prototype.listeners = function listeners(type) { - var evlistener; - var ret; - var events = this._events; - - if (!events) - ret = []; - else { - evlistener = events[type]; - if (!evlistener) - ret = []; - else if (typeof evlistener === 'function') - ret = [evlistener.listener || evlistener]; - else - ret = unwrapListeners(evlistener); - } - - return ret; - }; - - EventEmitter.listenerCount = function(emitter, type) { - if (typeof emitter.listenerCount === 'function') { - return emitter.listenerCount(type); - } else { - return listenerCount$1.call(emitter, type); - } - }; - - EventEmitter.prototype.listenerCount = listenerCount$1; - function listenerCount$1(type) { - var events = this._events; - - if (events) { - var evlistener = events[type]; - - if (typeof evlistener === 'function') { - return 1; - } else if (evlistener) { - return evlistener.length; - } - } - - return 0; - } - - EventEmitter.prototype.eventNames = function eventNames() { - return this._eventsCount > 0 ? Reflect.ownKeys(this._events) : []; - }; - - // About 1.5x faster than the two-arg version of Array#splice(). - function spliceOne(list, index) { - for (var i = index, k = i + 1, n = list.length; k < n; i += 1, k += 1) - list[i] = list[k]; - list.pop(); - } - - function arrayClone(arr, i) { - var copy = new Array(i); - while (i--) - copy[i] = arr[i]; - return copy; - } - - function unwrapListeners(arr) { - var ret = new Array(arr.length); - for (var i = 0; i < ret.length; ++i) { - ret[i] = arr[i].listener || arr[i]; - } - return ret; - } - - // shim for using process in browser - // based off https://github.com/defunctzombie/node-process/blob/master/browser.js - - function defaultSetTimout() { - throw new Error('setTimeout has not been defined'); - } - function defaultClearTimeout () { - throw new Error('clearTimeout has not been defined'); - } - var cachedSetTimeout = defaultSetTimout; - var cachedClearTimeout = defaultClearTimeout; - if (typeof global$1.setTimeout === 'function') { - cachedSetTimeout = setTimeout; - } - if (typeof global$1.clearTimeout === 'function') { - cachedClearTimeout = clearTimeout; - } - - function runTimeout(fun) { - if (cachedSetTimeout === setTimeout) { - //normal enviroments in sane situations - return setTimeout(fun, 0); - } - // if setTimeout wasn't available but was latter defined - if ((cachedSetTimeout === defaultSetTimout || !cachedSetTimeout) && setTimeout) { - cachedSetTimeout = setTimeout; - return setTimeout(fun, 0); - } - try { - // when when somebody has screwed with setTimeout but no I.E. maddness - return cachedSetTimeout(fun, 0); - } catch(e){ - try { - // When we are in I.E. but the script has been evaled so I.E. doesn't trust the global object when called normally - return cachedSetTimeout.call(null, fun, 0); - } catch(e){ - // same as above but when it's a version of I.E. that must have the global object for 'this', hopfully our context correct otherwise it will throw a global error - return cachedSetTimeout.call(this, fun, 0); - } - } - - - } - function runClearTimeout(marker) { - if (cachedClearTimeout === clearTimeout) { - //normal enviroments in sane situations - return clearTimeout(marker); - } - // if clearTimeout wasn't available but was latter defined - if ((cachedClearTimeout === defaultClearTimeout || !cachedClearTimeout) && clearTimeout) { - cachedClearTimeout = clearTimeout; - return clearTimeout(marker); - } - try { - // when when somebody has screwed with setTimeout but no I.E. maddness - return cachedClearTimeout(marker); - } catch (e){ - try { - // When we are in I.E. but the script has been evaled so I.E. doesn't trust the global object when called normally - return cachedClearTimeout.call(null, marker); - } catch (e){ - // same as above but when it's a version of I.E. that must have the global object for 'this', hopfully our context correct otherwise it will throw a global error. - // Some versions of I.E. have different rules for clearTimeout vs setTimeout - return cachedClearTimeout.call(this, marker); - } - } - - - - } - var queue = []; - var draining = false; - var currentQueue; - var queueIndex = -1; - - function cleanUpNextTick() { - if (!draining || !currentQueue) { - return; - } - draining = false; - if (currentQueue.length) { - queue = currentQueue.concat(queue); - } else { - queueIndex = -1; - } - if (queue.length) { - drainQueue(); - } - } - - function drainQueue() { - if (draining) { - return; - } - var timeout = runTimeout(cleanUpNextTick); - draining = true; - - var len = queue.length; - while(len) { - currentQueue = queue; - queue = []; - while (++queueIndex < len) { - if (currentQueue) { - currentQueue[queueIndex].run(); - } - } - queueIndex = -1; - len = queue.length; - } - currentQueue = null; - draining = false; - runClearTimeout(timeout); - } - function nextTick(fun) { - var args = new Array(arguments.length - 1); - if (arguments.length > 1) { - for (var i = 1; i < arguments.length; i++) { - args[i - 1] = arguments[i]; - } - } - queue.push(new Item(fun, args)); - if (queue.length === 1 && !draining) { - runTimeout(drainQueue); - } - } - // v8 likes predictible objects - function Item(fun, array) { - this.fun = fun; - this.array = array; - } - Item.prototype.run = function () { - this.fun.apply(null, this.array); - }; - - // from https://github.com/kumavis/browser-process-hrtime/blob/master/index.js - var performance = global$1.performance || {}; - performance.now || - performance.mozNow || - performance.msNow || - performance.oNow || - performance.webkitNow || - function(){ return (new Date()).getTime() }; - - var inherits; - if (typeof Object.create === 'function'){ - inherits = function inherits(ctor, superCtor) { - // implementation from standard node.js 'util' module - ctor.super_ = superCtor; - ctor.prototype = Object.create(superCtor.prototype, { - constructor: { - value: ctor, - enumerable: false, - writable: true, - configurable: true - } - }); - }; - } else { - inherits = function inherits(ctor, superCtor) { - ctor.super_ = superCtor; - var TempCtor = function () {}; - TempCtor.prototype = superCtor.prototype; - ctor.prototype = new TempCtor(); - ctor.prototype.constructor = ctor; - }; - } - var inherits$1 = inherits; - - var formatRegExp = /%[sdj%]/g; - function format(f) { - if (!isString(f)) { - var objects = []; - for (var i = 0; i < arguments.length; i++) { - objects.push(inspect(arguments[i])); - } - return objects.join(' '); - } - - var i = 1; - var args = arguments; - var len = args.length; - var str = String(f).replace(formatRegExp, function(x) { - if (x === '%%') return '%'; - if (i >= len) return x; - switch (x) { - case '%s': return String(args[i++]); - case '%d': return Number(args[i++]); - case '%j': - try { - return JSON.stringify(args[i++]); - } catch (_) { - return '[Circular]'; - } - default: - return x; - } - }); - for (var x = args[i]; i < len; x = args[++i]) { - if (isNull(x) || !isObject$1(x)) { - str += ' ' + x; - } else { - str += ' ' + inspect(x); - } - } - return str; - } - - // Mark that a method should not be used. - // Returns a modified function which warns once by default. - // If --no-deprecation is set, then it is a no-op. - function deprecate(fn, msg) { - // Allow for deprecating things in the process of starting up. - if (isUndefined(global$1.process)) { - return function() { - return deprecate(fn, msg).apply(this, arguments); - }; - } - - var warned = false; - function deprecated() { - if (!warned) { - { - console.error(msg); - } - warned = true; - } - return fn.apply(this, arguments); - } - - return deprecated; - } - - var debugs = {}; - var debugEnviron; - function debuglog(set) { - if (isUndefined(debugEnviron)) - debugEnviron = ''; - set = set.toUpperCase(); - if (!debugs[set]) { - if (new RegExp('\\b' + set + '\\b', 'i').test(debugEnviron)) { - var pid = 0; - debugs[set] = function() { - var msg = format.apply(null, arguments); - console.error('%s %d: %s', set, pid, msg); - }; - } else { - debugs[set] = function() {}; - } - } - return debugs[set]; - } - - /** - * Echos the value of a value. Trys to print the value out - * in the best way possible given the different types. - * - * @param {Object} obj The object to print out. - * @param {Object} opts Optional options object that alters the output. - */ - /* legacy: obj, showHidden, depth, colors*/ - function inspect(obj, opts) { - // default options - var ctx = { - seen: [], - stylize: stylizeNoColor - }; - // legacy... - if (arguments.length >= 3) ctx.depth = arguments[2]; - if (arguments.length >= 4) ctx.colors = arguments[3]; - if (isBoolean(opts)) { - // legacy... - ctx.showHidden = opts; - } else if (opts) { - // got an "options" object - _extend(ctx, opts); - } - // set default options - if (isUndefined(ctx.showHidden)) ctx.showHidden = false; - if (isUndefined(ctx.depth)) ctx.depth = 2; - if (isUndefined(ctx.colors)) ctx.colors = false; - if (isUndefined(ctx.customInspect)) ctx.customInspect = true; - if (ctx.colors) ctx.stylize = stylizeWithColor; - return formatValue(ctx, obj, ctx.depth); - } - - // http://en.wikipedia.org/wiki/ANSI_escape_code#graphics - inspect.colors = { - 'bold' : [1, 22], - 'italic' : [3, 23], - 'underline' : [4, 24], - 'inverse' : [7, 27], - 'white' : [37, 39], - 'grey' : [90, 39], - 'black' : [30, 39], - 'blue' : [34, 39], - 'cyan' : [36, 39], - 'green' : [32, 39], - 'magenta' : [35, 39], - 'red' : [31, 39], - 'yellow' : [33, 39] - }; - - // Don't use 'blue' not visible on cmd.exe - inspect.styles = { - 'special': 'cyan', - 'number': 'yellow', - 'boolean': 'yellow', - 'undefined': 'grey', - 'null': 'bold', - 'string': 'green', - 'date': 'magenta', - // "name": intentionally not styling - 'regexp': 'red' - }; - - - function stylizeWithColor(str, styleType) { - var style = inspect.styles[styleType]; - - if (style) { - return '\u001b[' + inspect.colors[style][0] + 'm' + str + - '\u001b[' + inspect.colors[style][1] + 'm'; - } else { - return str; - } - } - - - function stylizeNoColor(str, styleType) { - return str; - } - - - function arrayToHash(array) { - var hash = {}; - - array.forEach(function(val, idx) { - hash[val] = true; - }); - - return hash; - } - - - function formatValue(ctx, value, recurseTimes) { - // Provide a hook for user-specified inspect functions. - // Check that value is an object with an inspect function on it - if (ctx.customInspect && - value && - isFunction(value.inspect) && - // Filter out the util module, it's inspect function is special - value.inspect !== inspect && - // Also filter out any prototype objects using the circular check. - !(value.constructor && value.constructor.prototype === value)) { - var ret = value.inspect(recurseTimes, ctx); - if (!isString(ret)) { - ret = formatValue(ctx, ret, recurseTimes); - } - return ret; - } - - // Primitive types cannot have properties - var primitive = formatPrimitive(ctx, value); - if (primitive) { - return primitive; - } - - // Look up the keys of the object. - var keys = Object.keys(value); - var visibleKeys = arrayToHash(keys); - - if (ctx.showHidden) { - keys = Object.getOwnPropertyNames(value); - } - - // IE doesn't make error fields non-enumerable - // http://msdn.microsoft.com/en-us/library/ie/dww52sbt(v=vs.94).aspx - if (isError(value) - && (keys.indexOf('message') >= 0 || keys.indexOf('description') >= 0)) { - return formatError(value); - } - - // Some type of object without properties can be shortcutted. - if (keys.length === 0) { - if (isFunction(value)) { - var name = value.name ? ': ' + value.name : ''; - return ctx.stylize('[Function' + name + ']', 'special'); - } - if (isRegExp(value)) { - return ctx.stylize(RegExp.prototype.toString.call(value), 'regexp'); - } - if (isDate(value)) { - return ctx.stylize(Date.prototype.toString.call(value), 'date'); - } - if (isError(value)) { - return formatError(value); - } - } - - var base = '', array = false, braces = ['{', '}']; - - // Make Array say that they are Array - if (isArray(value)) { - array = true; - braces = ['[', ']']; - } - - // Make functions say that they are functions - if (isFunction(value)) { - var n = value.name ? ': ' + value.name : ''; - base = ' [Function' + n + ']'; - } - - // Make RegExps say that they are RegExps - if (isRegExp(value)) { - base = ' ' + RegExp.prototype.toString.call(value); - } - - // Make dates with properties first say the date - if (isDate(value)) { - base = ' ' + Date.prototype.toUTCString.call(value); - } - - // Make error with message first say the error - if (isError(value)) { - base = ' ' + formatError(value); - } - - if (keys.length === 0 && (!array || value.length == 0)) { - return braces[0] + base + braces[1]; - } - - if (recurseTimes < 0) { - if (isRegExp(value)) { - return ctx.stylize(RegExp.prototype.toString.call(value), 'regexp'); - } else { - return ctx.stylize('[Object]', 'special'); - } - } - - ctx.seen.push(value); - - var output; - if (array) { - output = formatArray(ctx, value, recurseTimes, visibleKeys, keys); - } else { - output = keys.map(function(key) { - return formatProperty(ctx, value, recurseTimes, visibleKeys, key, array); - }); - } - - ctx.seen.pop(); - - return reduceToSingleString(output, base, braces); - } - - - function formatPrimitive(ctx, value) { - if (isUndefined(value)) - return ctx.stylize('undefined', 'undefined'); - if (isString(value)) { - var simple = '\'' + JSON.stringify(value).replace(/^"|"$/g, '') - .replace(/'/g, "\\'") - .replace(/\\"/g, '"') + '\''; - return ctx.stylize(simple, 'string'); - } - if (isNumber(value)) - return ctx.stylize('' + value, 'number'); - if (isBoolean(value)) - return ctx.stylize('' + value, 'boolean'); - // For some reason typeof null is "object", so special case here. - if (isNull(value)) - return ctx.stylize('null', 'null'); - } - - - function formatError(value) { - return '[' + Error.prototype.toString.call(value) + ']'; - } - - - function formatArray(ctx, value, recurseTimes, visibleKeys, keys) { - var output = []; - for (var i = 0, l = value.length; i < l; ++i) { - if (hasOwnProperty(value, String(i))) { - output.push(formatProperty(ctx, value, recurseTimes, visibleKeys, - String(i), true)); - } else { - output.push(''); - } - } - keys.forEach(function(key) { - if (!key.match(/^\d+$/)) { - output.push(formatProperty(ctx, value, recurseTimes, visibleKeys, - key, true)); - } - }); - return output; - } - - - function formatProperty(ctx, value, recurseTimes, visibleKeys, key, array) { - var name, str, desc; - desc = Object.getOwnPropertyDescriptor(value, key) || { value: value[key] }; - if (desc.get) { - if (desc.set) { - str = ctx.stylize('[Getter/Setter]', 'special'); - } else { - str = ctx.stylize('[Getter]', 'special'); - } - } else { - if (desc.set) { - str = ctx.stylize('[Setter]', 'special'); - } - } - if (!hasOwnProperty(visibleKeys, key)) { - name = '[' + key + ']'; - } - if (!str) { - if (ctx.seen.indexOf(desc.value) < 0) { - if (isNull(recurseTimes)) { - str = formatValue(ctx, desc.value, null); - } else { - str = formatValue(ctx, desc.value, recurseTimes - 1); - } - if (str.indexOf('\n') > -1) { - if (array) { - str = str.split('\n').map(function(line) { - return ' ' + line; - }).join('\n').substr(2); - } else { - str = '\n' + str.split('\n').map(function(line) { - return ' ' + line; - }).join('\n'); - } - } - } else { - str = ctx.stylize('[Circular]', 'special'); - } - } - if (isUndefined(name)) { - if (array && key.match(/^\d+$/)) { - return str; - } - name = JSON.stringify('' + key); - if (name.match(/^"([a-zA-Z_][a-zA-Z_0-9]*)"$/)) { - name = name.substr(1, name.length - 2); - name = ctx.stylize(name, 'name'); - } else { - name = name.replace(/'/g, "\\'") - .replace(/\\"/g, '"') - .replace(/(^"|"$)/g, "'"); - name = ctx.stylize(name, 'string'); - } - } - - return name + ': ' + str; - } - - - function reduceToSingleString(output, base, braces) { - var length = output.reduce(function(prev, cur) { - if (cur.indexOf('\n') >= 0) ; - return prev + cur.replace(/\u001b\[\d\d?m/g, '').length + 1; - }, 0); - - if (length > 60) { - return braces[0] + - (base === '' ? '' : base + '\n ') + - ' ' + - output.join(',\n ') + - ' ' + - braces[1]; - } - - return braces[0] + base + ' ' + output.join(', ') + ' ' + braces[1]; - } - - - // NOTE: These type checking functions intentionally don't use `instanceof` - // because it is fragile and can be easily faked with `Object.create()`. - function isArray(ar) { - return Array.isArray(ar); - } - - function isBoolean(arg) { - return typeof arg === 'boolean'; - } - - function isNull(arg) { - return arg === null; - } - - function isNumber(arg) { - return typeof arg === 'number'; - } - - function isString(arg) { - return typeof arg === 'string'; - } - - function isUndefined(arg) { - return arg === void 0; - } - - function isRegExp(re) { - return isObject$1(re) && objectToString(re) === '[object RegExp]'; - } - - function isObject$1(arg) { - return typeof arg === 'object' && arg !== null; - } - - function isDate(d) { - return isObject$1(d) && objectToString(d) === '[object Date]'; - } - - function isError(e) { - return isObject$1(e) && - (objectToString(e) === '[object Error]' || e instanceof Error); - } - - function isFunction(arg) { - return typeof arg === 'function'; - } - - function objectToString(o) { - return Object.prototype.toString.call(o); - } - - function _extend(origin, add) { - // Don't do anything if add isn't an object - if (!add || !isObject$1(add)) return origin; - - var keys = Object.keys(add); - var i = keys.length; - while (i--) { - origin[keys[i]] = add[keys[i]]; - } - return origin; - } - function hasOwnProperty(obj, prop) { - return Object.prototype.hasOwnProperty.call(obj, prop); - } - - function BufferList() { - this.head = null; - this.tail = null; - this.length = 0; - } - - BufferList.prototype.push = function (v) { - var entry = { data: v, next: null }; - if (this.length > 0) this.tail.next = entry;else this.head = entry; - this.tail = entry; - ++this.length; - }; - - BufferList.prototype.unshift = function (v) { - var entry = { data: v, next: this.head }; - if (this.length === 0) this.tail = entry; - this.head = entry; - ++this.length; - }; - - BufferList.prototype.shift = function () { - if (this.length === 0) return; - var ret = this.head.data; - if (this.length === 1) this.head = this.tail = null;else this.head = this.head.next; - --this.length; - return ret; - }; - - BufferList.prototype.clear = function () { - this.head = this.tail = null; - this.length = 0; - }; - - BufferList.prototype.join = function (s) { - if (this.length === 0) return ''; - var p = this.head; - var ret = '' + p.data; - while (p = p.next) { - ret += s + p.data; - }return ret; - }; - - BufferList.prototype.concat = function (n) { - if (this.length === 0) return Buffer.alloc(0); - if (this.length === 1) return this.head.data; - var ret = Buffer.allocUnsafe(n >>> 0); - var p = this.head; - var i = 0; - while (p) { - p.data.copy(ret, i); - i += p.data.length; - p = p.next; - } - return ret; - }; - - // Copyright Joyent, Inc. and other Node contributors. - var isBufferEncoding = Buffer.isEncoding - || function(encoding) { - switch (encoding && encoding.toLowerCase()) { - case 'hex': case 'utf8': case 'utf-8': case 'ascii': case 'binary': case 'base64': case 'ucs2': case 'ucs-2': case 'utf16le': case 'utf-16le': case 'raw': return true; - default: return false; - } - }; - - - function assertEncoding(encoding) { - if (encoding && !isBufferEncoding(encoding)) { - throw new Error('Unknown encoding: ' + encoding); - } - } - - // StringDecoder provides an interface for efficiently splitting a series of - // buffers into a series of JS strings without breaking apart multi-byte - // characters. CESU-8 is handled as part of the UTF-8 encoding. - // - // @TODO Handling all encodings inside a single object makes it very difficult - // to reason about this code, so it should be split up in the future. - // @TODO There should be a utf8-strict encoding that rejects invalid UTF-8 code - // points as used by CESU-8. - function StringDecoder(encoding) { - this.encoding = (encoding || 'utf8').toLowerCase().replace(/[-_]/, ''); - assertEncoding(encoding); - switch (this.encoding) { - case 'utf8': - // CESU-8 represents each of Surrogate Pair by 3-bytes - this.surrogateSize = 3; - break; - case 'ucs2': - case 'utf16le': - // UTF-16 represents each of Surrogate Pair by 2-bytes - this.surrogateSize = 2; - this.detectIncompleteChar = utf16DetectIncompleteChar; - break; - case 'base64': - // Base-64 stores 3 bytes in 4 chars, and pads the remainder. - this.surrogateSize = 3; - this.detectIncompleteChar = base64DetectIncompleteChar; - break; - default: - this.write = passThroughWrite; - return; - } - - // Enough space to store all bytes of a single character. UTF-8 needs 4 - // bytes, but CESU-8 may require up to 6 (3 bytes per surrogate). - this.charBuffer = new Buffer(6); - // Number of bytes received for the current incomplete multi-byte character. - this.charReceived = 0; - // Number of bytes expected for the current incomplete multi-byte character. - this.charLength = 0; - } - - // write decodes the given buffer and returns it as JS string that is - // guaranteed to not contain any partial multi-byte characters. Any partial - // character found at the end of the buffer is buffered up, and will be - // returned when calling write again with the remaining bytes. - // - // Note: Converting a Buffer containing an orphan surrogate to a String - // currently works, but converting a String to a Buffer (via `new Buffer`, or - // Buffer#write) will replace incomplete surrogates with the unicode - // replacement character. See https://codereview.chromium.org/121173009/ . - StringDecoder.prototype.write = function(buffer) { - var charStr = ''; - // if our last write ended with an incomplete multibyte character - while (this.charLength) { - // determine how many remaining bytes this buffer has to offer for this char - var available = (buffer.length >= this.charLength - this.charReceived) ? - this.charLength - this.charReceived : - buffer.length; - - // add the new bytes to the char buffer - buffer.copy(this.charBuffer, this.charReceived, 0, available); - this.charReceived += available; - - if (this.charReceived < this.charLength) { - // still not enough chars in this buffer? wait for more ... - return ''; - } - - // remove bytes belonging to the current character from the buffer - buffer = buffer.slice(available, buffer.length); - - // get the character that was split - charStr = this.charBuffer.slice(0, this.charLength).toString(this.encoding); - - // CESU-8: lead surrogate (D800-DBFF) is also the incomplete character - var charCode = charStr.charCodeAt(charStr.length - 1); - if (charCode >= 0xD800 && charCode <= 0xDBFF) { - this.charLength += this.surrogateSize; - charStr = ''; - continue; - } - this.charReceived = this.charLength = 0; - - // if there are no more bytes in this buffer, just emit our char - if (buffer.length === 0) { - return charStr; - } - break; - } - - // determine and set charLength / charReceived - this.detectIncompleteChar(buffer); - - var end = buffer.length; - if (this.charLength) { - // buffer the incomplete character bytes we got - buffer.copy(this.charBuffer, 0, buffer.length - this.charReceived, end); - end -= this.charReceived; - } - - charStr += buffer.toString(this.encoding, 0, end); - - var end = charStr.length - 1; - var charCode = charStr.charCodeAt(end); - // CESU-8: lead surrogate (D800-DBFF) is also the incomplete character - if (charCode >= 0xD800 && charCode <= 0xDBFF) { - var size = this.surrogateSize; - this.charLength += size; - this.charReceived += size; - this.charBuffer.copy(this.charBuffer, size, 0, size); - buffer.copy(this.charBuffer, 0, 0, size); - return charStr.substring(0, end); - } - - // or just emit the charStr - return charStr; - }; - - // detectIncompleteChar determines if there is an incomplete UTF-8 character at - // the end of the given buffer. If so, it sets this.charLength to the byte - // length that character, and sets this.charReceived to the number of bytes - // that are available for this character. - StringDecoder.prototype.detectIncompleteChar = function(buffer) { - // determine how many bytes we have to check at the end of this buffer - var i = (buffer.length >= 3) ? 3 : buffer.length; - - // Figure out if one of the last i bytes of our buffer announces an - // incomplete char. - for (; i > 0; i--) { - var c = buffer[buffer.length - i]; - - // See http://en.wikipedia.org/wiki/UTF-8#Description - - // 110XXXXX - if (i == 1 && c >> 5 == 0x06) { - this.charLength = 2; - break; - } - - // 1110XXXX - if (i <= 2 && c >> 4 == 0x0E) { - this.charLength = 3; - break; - } - - // 11110XXX - if (i <= 3 && c >> 3 == 0x1E) { - this.charLength = 4; - break; - } - } - this.charReceived = i; - }; - - StringDecoder.prototype.end = function(buffer) { - var res = ''; - if (buffer && buffer.length) - res = this.write(buffer); - - if (this.charReceived) { - var cr = this.charReceived; - var buf = this.charBuffer; - var enc = this.encoding; - res += buf.slice(0, cr).toString(enc); - } - - return res; - }; - - function passThroughWrite(buffer) { - return buffer.toString(this.encoding); - } - - function utf16DetectIncompleteChar(buffer) { - this.charReceived = buffer.length % 2; - this.charLength = this.charReceived ? 2 : 0; - } - - function base64DetectIncompleteChar(buffer) { - this.charReceived = buffer.length % 3; - this.charLength = this.charReceived ? 3 : 0; - } - - Readable.ReadableState = ReadableState; - - var debug = debuglog('stream'); - inherits$1(Readable, EventEmitter); - - function prependListener(emitter, event, fn) { - // Sadly this is not cacheable as some libraries bundle their own - // event emitter implementation with them. - if (typeof emitter.prependListener === 'function') { - return emitter.prependListener(event, fn); - } else { - // This is a hack to make sure that our error handler is attached before any - // userland ones. NEVER DO THIS. This is here only because this code needs - // to continue to work with older versions of Node.js that do not include - // the prependListener() method. The goal is to eventually remove this hack. - if (!emitter._events || !emitter._events[event]) - emitter.on(event, fn); - else if (Array.isArray(emitter._events[event])) - emitter._events[event].unshift(fn); - else - emitter._events[event] = [fn, emitter._events[event]]; - } - } - function listenerCount (emitter, type) { - return emitter.listeners(type).length; - } - function ReadableState(options, stream) { - - options = options || {}; - - // object stream flag. Used to make read(n) ignore n and to - // make all the buffer merging and length checks go away - this.objectMode = !!options.objectMode; - - if (stream instanceof Duplex) this.objectMode = this.objectMode || !!options.readableObjectMode; - - // the point at which it stops calling _read() to fill the buffer - // Note: 0 is a valid value, means "don't call _read preemptively ever" - var hwm = options.highWaterMark; - var defaultHwm = this.objectMode ? 16 : 16 * 1024; - this.highWaterMark = hwm || hwm === 0 ? hwm : defaultHwm; - - // cast to ints. - this.highWaterMark = ~ ~this.highWaterMark; - - // A linked list is used to store data chunks instead of an array because the - // linked list can remove elements from the beginning faster than - // array.shift() - this.buffer = new BufferList(); - this.length = 0; - this.pipes = null; - this.pipesCount = 0; - this.flowing = null; - this.ended = false; - this.endEmitted = false; - this.reading = false; - - // a flag to be able to tell if the onwrite cb is called immediately, - // or on a later tick. We set this to true at first, because any - // actions that shouldn't happen until "later" should generally also - // not happen before the first write call. - this.sync = true; - - // whenever we return null, then we set a flag to say - // that we're awaiting a 'readable' event emission. - this.needReadable = false; - this.emittedReadable = false; - this.readableListening = false; - this.resumeScheduled = false; - - // Crypto is kind of old and crusty. Historically, its default string - // encoding is 'binary' so we have to make this configurable. - // Everything else in the universe uses 'utf8', though. - this.defaultEncoding = options.defaultEncoding || 'utf8'; - - // when piping, we only care about 'readable' events that happen - // after read()ing all the bytes and not getting any pushback. - this.ranOut = false; - - // the number of writers that are awaiting a drain event in .pipe()s - this.awaitDrain = 0; - - // if true, a maybeReadMore has been scheduled - this.readingMore = false; - - this.decoder = null; - this.encoding = null; - if (options.encoding) { - this.decoder = new StringDecoder(options.encoding); - this.encoding = options.encoding; - } - } - function Readable(options) { - - if (!(this instanceof Readable)) return new Readable(options); - - this._readableState = new ReadableState(options, this); - - // legacy - this.readable = true; - - if (options && typeof options.read === 'function') this._read = options.read; - - EventEmitter.call(this); - } - - // Manually shove something into the read() buffer. - // This returns true if the highWaterMark has not been hit yet, - // similar to how Writable.write() returns true if you should - // write() some more. - Readable.prototype.push = function (chunk, encoding) { - var state = this._readableState; - - if (!state.objectMode && typeof chunk === 'string') { - encoding = encoding || state.defaultEncoding; - if (encoding !== state.encoding) { - chunk = Buffer.from(chunk, encoding); - encoding = ''; - } - } - - return readableAddChunk(this, state, chunk, encoding, false); - }; - - // Unshift should *always* be something directly out of read() - Readable.prototype.unshift = function (chunk) { - var state = this._readableState; - return readableAddChunk(this, state, chunk, '', true); - }; - - Readable.prototype.isPaused = function () { - return this._readableState.flowing === false; - }; - - function readableAddChunk(stream, state, chunk, encoding, addToFront) { - var er = chunkInvalid(state, chunk); - if (er) { - stream.emit('error', er); - } else if (chunk === null) { - state.reading = false; - onEofChunk(stream, state); - } else if (state.objectMode || chunk && chunk.length > 0) { - if (state.ended && !addToFront) { - var e = new Error('stream.push() after EOF'); - stream.emit('error', e); - } else if (state.endEmitted && addToFront) { - var _e = new Error('stream.unshift() after end event'); - stream.emit('error', _e); - } else { - var skipAdd; - if (state.decoder && !addToFront && !encoding) { - chunk = state.decoder.write(chunk); - skipAdd = !state.objectMode && chunk.length === 0; - } - - if (!addToFront) state.reading = false; - - // Don't add to the buffer if we've decoded to an empty string chunk and - // we're not in object mode - if (!skipAdd) { - // if we want the data now, just emit it. - if (state.flowing && state.length === 0 && !state.sync) { - stream.emit('data', chunk); - stream.read(0); - } else { - // update the buffer info. - state.length += state.objectMode ? 1 : chunk.length; - if (addToFront) state.buffer.unshift(chunk);else state.buffer.push(chunk); - - if (state.needReadable) emitReadable(stream); - } - } - - maybeReadMore(stream, state); - } - } else if (!addToFront) { - state.reading = false; - } - - return needMoreData(state); - } - - // if it's past the high water mark, we can push in some more. - // Also, if we have no data yet, we can stand some - // more bytes. This is to work around cases where hwm=0, - // such as the repl. Also, if the push() triggered a - // readable event, and the user called read(largeNumber) such that - // needReadable was set, then we ought to push more, so that another - // 'readable' event will be triggered. - function needMoreData(state) { - return !state.ended && (state.needReadable || state.length < state.highWaterMark || state.length === 0); - } - - // backwards compatibility. - Readable.prototype.setEncoding = function (enc) { - this._readableState.decoder = new StringDecoder(enc); - this._readableState.encoding = enc; - return this; - }; - - // Don't raise the hwm > 8MB - var MAX_HWM = 0x800000; - function computeNewHighWaterMark(n) { - if (n >= MAX_HWM) { - n = MAX_HWM; - } else { - // Get the next highest power of 2 to prevent increasing hwm excessively in - // tiny amounts - n--; - n |= n >>> 1; - n |= n >>> 2; - n |= n >>> 4; - n |= n >>> 8; - n |= n >>> 16; - n++; - } - return n; - } - - // This function is designed to be inlinable, so please take care when making - // changes to the function body. - function howMuchToRead(n, state) { - if (n <= 0 || state.length === 0 && state.ended) return 0; - if (state.objectMode) return 1; - if (n !== n) { - // Only flow one buffer at a time - if (state.flowing && state.length) return state.buffer.head.data.length;else return state.length; - } - // If we're asking for more than the current hwm, then raise the hwm. - if (n > state.highWaterMark) state.highWaterMark = computeNewHighWaterMark(n); - if (n <= state.length) return n; - // Don't have enough - if (!state.ended) { - state.needReadable = true; - return 0; - } - return state.length; - } - - // you can override either this method, or the async _read(n) below. - Readable.prototype.read = function (n) { - debug('read', n); - n = parseInt(n, 10); - var state = this._readableState; - var nOrig = n; - - if (n !== 0) state.emittedReadable = false; - - // if we're doing read(0) to trigger a readable event, but we - // already have a bunch of data in the buffer, then just trigger - // the 'readable' event and move on. - if (n === 0 && state.needReadable && (state.length >= state.highWaterMark || state.ended)) { - debug('read: emitReadable', state.length, state.ended); - if (state.length === 0 && state.ended) endReadable(this);else emitReadable(this); - return null; - } - - n = howMuchToRead(n, state); - - // if we've ended, and we're now clear, then finish it up. - if (n === 0 && state.ended) { - if (state.length === 0) endReadable(this); - return null; - } - - // All the actual chunk generation logic needs to be - // *below* the call to _read. The reason is that in certain - // synthetic stream cases, such as passthrough streams, _read - // may be a completely synchronous operation which may change - // the state of the read buffer, providing enough data when - // before there was *not* enough. - // - // So, the steps are: - // 1. Figure out what the state of things will be after we do - // a read from the buffer. - // - // 2. If that resulting state will trigger a _read, then call _read. - // Note that this may be asynchronous, or synchronous. Yes, it is - // deeply ugly to write APIs this way, but that still doesn't mean - // that the Readable class should behave improperly, as streams are - // designed to be sync/async agnostic. - // Take note if the _read call is sync or async (ie, if the read call - // has returned yet), so that we know whether or not it's safe to emit - // 'readable' etc. - // - // 3. Actually pull the requested chunks out of the buffer and return. - - // if we need a readable event, then we need to do some reading. - var doRead = state.needReadable; - debug('need readable', doRead); - - // if we currently have less than the highWaterMark, then also read some - if (state.length === 0 || state.length - n < state.highWaterMark) { - doRead = true; - debug('length less than watermark', doRead); - } - - // however, if we've ended, then there's no point, and if we're already - // reading, then it's unnecessary. - if (state.ended || state.reading) { - doRead = false; - debug('reading or ended', doRead); - } else if (doRead) { - debug('do read'); - state.reading = true; - state.sync = true; - // if the length is currently zero, then we *need* a readable event. - if (state.length === 0) state.needReadable = true; - // call internal read method - this._read(state.highWaterMark); - state.sync = false; - // If _read pushed data synchronously, then `reading` will be false, - // and we need to re-evaluate how much data we can return to the user. - if (!state.reading) n = howMuchToRead(nOrig, state); - } - - var ret; - if (n > 0) ret = fromList(n, state);else ret = null; - - if (ret === null) { - state.needReadable = true; - n = 0; - } else { - state.length -= n; - } - - if (state.length === 0) { - // If we have nothing in the buffer, then we want to know - // as soon as we *do* get something into the buffer. - if (!state.ended) state.needReadable = true; - - // If we tried to read() past the EOF, then emit end on the next tick. - if (nOrig !== n && state.ended) endReadable(this); - } - - if (ret !== null) this.emit('data', ret); - - return ret; - }; - - function chunkInvalid(state, chunk) { - var er = null; - if (!isBuffer(chunk) && typeof chunk !== 'string' && chunk !== null && chunk !== undefined && !state.objectMode) { - er = new TypeError('Invalid non-string/buffer chunk'); - } - return er; - } - - function onEofChunk(stream, state) { - if (state.ended) return; - if (state.decoder) { - var chunk = state.decoder.end(); - if (chunk && chunk.length) { - state.buffer.push(chunk); - state.length += state.objectMode ? 1 : chunk.length; - } - } - state.ended = true; - - // emit 'readable' now to make sure it gets picked up. - emitReadable(stream); - } - - // Don't emit readable right away in sync mode, because this can trigger - // another read() call => stack overflow. This way, it might trigger - // a nextTick recursion warning, but that's not so bad. - function emitReadable(stream) { - var state = stream._readableState; - state.needReadable = false; - if (!state.emittedReadable) { - debug('emitReadable', state.flowing); - state.emittedReadable = true; - if (state.sync) nextTick(emitReadable_, stream);else emitReadable_(stream); - } - } - - function emitReadable_(stream) { - debug('emit readable'); - stream.emit('readable'); - flow(stream); - } - - // at this point, the user has presumably seen the 'readable' event, - // and called read() to consume some data. that may have triggered - // in turn another _read(n) call, in which case reading = true if - // it's in progress. - // However, if we're not ended, or reading, and the length < hwm, - // then go ahead and try to read some more preemptively. - function maybeReadMore(stream, state) { - if (!state.readingMore) { - state.readingMore = true; - nextTick(maybeReadMore_, stream, state); - } - } - - function maybeReadMore_(stream, state) { - var len = state.length; - while (!state.reading && !state.flowing && !state.ended && state.length < state.highWaterMark) { - debug('maybeReadMore read 0'); - stream.read(0); - if (len === state.length) - // didn't get any data, stop spinning. - break;else len = state.length; - } - state.readingMore = false; - } - - // abstract method. to be overridden in specific implementation classes. - // call cb(er, data) where data is <= n in length. - // for virtual (non-string, non-buffer) streams, "length" is somewhat - // arbitrary, and perhaps not very meaningful. - Readable.prototype._read = function (n) { - this.emit('error', new Error('not implemented')); - }; - - Readable.prototype.pipe = function (dest, pipeOpts) { - var src = this; - var state = this._readableState; - - switch (state.pipesCount) { - case 0: - state.pipes = dest; - break; - case 1: - state.pipes = [state.pipes, dest]; - break; - default: - state.pipes.push(dest); - break; - } - state.pipesCount += 1; - debug('pipe count=%d opts=%j', state.pipesCount, pipeOpts); - - var doEnd = (!pipeOpts || pipeOpts.end !== false); - - var endFn = doEnd ? onend : cleanup; - if (state.endEmitted) nextTick(endFn);else src.once('end', endFn); - - dest.on('unpipe', onunpipe); - function onunpipe(readable) { - debug('onunpipe'); - if (readable === src) { - cleanup(); - } - } - - function onend() { - debug('onend'); - dest.end(); - } - - // when the dest drains, it reduces the awaitDrain counter - // on the source. This would be more elegant with a .once() - // handler in flow(), but adding and removing repeatedly is - // too slow. - var ondrain = pipeOnDrain(src); - dest.on('drain', ondrain); - - var cleanedUp = false; - function cleanup() { - debug('cleanup'); - // cleanup event handlers once the pipe is broken - dest.removeListener('close', onclose); - dest.removeListener('finish', onfinish); - dest.removeListener('drain', ondrain); - dest.removeListener('error', onerror); - dest.removeListener('unpipe', onunpipe); - src.removeListener('end', onend); - src.removeListener('end', cleanup); - src.removeListener('data', ondata); - - cleanedUp = true; - - // if the reader is waiting for a drain event from this - // specific writer, then it would cause it to never start - // flowing again. - // So, if this is awaiting a drain, then we just call it now. - // If we don't know, then assume that we are waiting for one. - if (state.awaitDrain && (!dest._writableState || dest._writableState.needDrain)) ondrain(); - } - - // If the user pushes more data while we're writing to dest then we'll end up - // in ondata again. However, we only want to increase awaitDrain once because - // dest will only emit one 'drain' event for the multiple writes. - // => Introduce a guard on increasing awaitDrain. - var increasedAwaitDrain = false; - src.on('data', ondata); - function ondata(chunk) { - debug('ondata'); - increasedAwaitDrain = false; - var ret = dest.write(chunk); - if (false === ret && !increasedAwaitDrain) { - // If the user unpiped during `dest.write()`, it is possible - // to get stuck in a permanently paused state if that write - // also returned false. - // => Check whether `dest` is still a piping destination. - if ((state.pipesCount === 1 && state.pipes === dest || state.pipesCount > 1 && indexOf(state.pipes, dest) !== -1) && !cleanedUp) { - debug('false write response, pause', src._readableState.awaitDrain); - src._readableState.awaitDrain++; - increasedAwaitDrain = true; - } - src.pause(); - } - } - - // if the dest has an error, then stop piping into it. - // however, don't suppress the throwing behavior for this. - function onerror(er) { - debug('onerror', er); - unpipe(); - dest.removeListener('error', onerror); - if (listenerCount(dest, 'error') === 0) dest.emit('error', er); - } - - // Make sure our error handler is attached before userland ones. - prependListener(dest, 'error', onerror); - - // Both close and finish should trigger unpipe, but only once. - function onclose() { - dest.removeListener('finish', onfinish); - unpipe(); - } - dest.once('close', onclose); - function onfinish() { - debug('onfinish'); - dest.removeListener('close', onclose); - unpipe(); - } - dest.once('finish', onfinish); - - function unpipe() { - debug('unpipe'); - src.unpipe(dest); - } - - // tell the dest that it's being piped to - dest.emit('pipe', src); - - // start the flow if it hasn't been started already. - if (!state.flowing) { - debug('pipe resume'); - src.resume(); - } - - return dest; - }; - - function pipeOnDrain(src) { - return function () { - var state = src._readableState; - debug('pipeOnDrain', state.awaitDrain); - if (state.awaitDrain) state.awaitDrain--; - if (state.awaitDrain === 0 && src.listeners('data').length) { - state.flowing = true; - flow(src); - } - }; - } - - Readable.prototype.unpipe = function (dest) { - var state = this._readableState; - - // if we're not piping anywhere, then do nothing. - if (state.pipesCount === 0) return this; - - // just one destination. most common case. - if (state.pipesCount === 1) { - // passed in one, but it's not the right one. - if (dest && dest !== state.pipes) return this; - - if (!dest) dest = state.pipes; - - // got a match. - state.pipes = null; - state.pipesCount = 0; - state.flowing = false; - if (dest) dest.emit('unpipe', this); - return this; - } - - // slow case. multiple pipe destinations. - - if (!dest) { - // remove all. - var dests = state.pipes; - var len = state.pipesCount; - state.pipes = null; - state.pipesCount = 0; - state.flowing = false; - - for (var _i = 0; _i < len; _i++) { - dests[_i].emit('unpipe', this); - }return this; - } - - // try to find the right one. - var i = indexOf(state.pipes, dest); - if (i === -1) return this; - - state.pipes.splice(i, 1); - state.pipesCount -= 1; - if (state.pipesCount === 1) state.pipes = state.pipes[0]; - - dest.emit('unpipe', this); - - return this; - }; - - // set up data events if they are asked for - // Ensure readable listeners eventually get something - Readable.prototype.on = function (ev, fn) { - var res = EventEmitter.prototype.on.call(this, ev, fn); - - if (ev === 'data') { - // Start flowing on next tick if stream isn't explicitly paused - if (this._readableState.flowing !== false) this.resume(); - } else if (ev === 'readable') { - var state = this._readableState; - if (!state.endEmitted && !state.readableListening) { - state.readableListening = state.needReadable = true; - state.emittedReadable = false; - if (!state.reading) { - nextTick(nReadingNextTick, this); - } else if (state.length) { - emitReadable(this); - } - } - } - - return res; - }; - Readable.prototype.addListener = Readable.prototype.on; - - function nReadingNextTick(self) { - debug('readable nexttick read 0'); - self.read(0); - } - - // pause() and resume() are remnants of the legacy readable stream API - // If the user uses them, then switch into old mode. - Readable.prototype.resume = function () { - var state = this._readableState; - if (!state.flowing) { - debug('resume'); - state.flowing = true; - resume(this, state); - } - return this; - }; - - function resume(stream, state) { - if (!state.resumeScheduled) { - state.resumeScheduled = true; - nextTick(resume_, stream, state); - } - } - - function resume_(stream, state) { - if (!state.reading) { - debug('resume read 0'); - stream.read(0); - } - - state.resumeScheduled = false; - state.awaitDrain = 0; - stream.emit('resume'); - flow(stream); - if (state.flowing && !state.reading) stream.read(0); - } - - Readable.prototype.pause = function () { - debug('call pause flowing=%j', this._readableState.flowing); - if (false !== this._readableState.flowing) { - debug('pause'); - this._readableState.flowing = false; - this.emit('pause'); - } - return this; - }; - - function flow(stream) { - var state = stream._readableState; - debug('flow', state.flowing); - while (state.flowing && stream.read() !== null) {} - } - - // wrap an old-style stream as the async data source. - // This is *not* part of the readable stream interface. - // It is an ugly unfortunate mess of history. - Readable.prototype.wrap = function (stream) { - var state = this._readableState; - var paused = false; - - var self = this; - stream.on('end', function () { - debug('wrapped end'); - if (state.decoder && !state.ended) { - var chunk = state.decoder.end(); - if (chunk && chunk.length) self.push(chunk); - } - - self.push(null); - }); - - stream.on('data', function (chunk) { - debug('wrapped data'); - if (state.decoder) chunk = state.decoder.write(chunk); - - // don't skip over falsy values in objectMode - if (state.objectMode && (chunk === null || chunk === undefined)) return;else if (!state.objectMode && (!chunk || !chunk.length)) return; - - var ret = self.push(chunk); - if (!ret) { - paused = true; - stream.pause(); - } - }); - - // proxy all the other methods. - // important when wrapping filters and duplexes. - for (var i in stream) { - if (this[i] === undefined && typeof stream[i] === 'function') { - this[i] = function (method) { - return function () { - return stream[method].apply(stream, arguments); - }; - }(i); - } - } - - // proxy certain important events. - var events = ['error', 'close', 'destroy', 'pause', 'resume']; - forEach(events, function (ev) { - stream.on(ev, self.emit.bind(self, ev)); - }); - - // when we try to consume some more bytes, simply unpause the - // underlying stream. - self._read = function (n) { - debug('wrapped _read', n); - if (paused) { - paused = false; - stream.resume(); - } - }; - - return self; - }; - - // exposed for testing purposes only. - Readable._fromList = fromList; - - // Pluck off n bytes from an array of buffers. - // Length is the combined lengths of all the buffers in the list. - // This function is designed to be inlinable, so please take care when making - // changes to the function body. - function fromList(n, state) { - // nothing buffered - if (state.length === 0) return null; - - var ret; - if (state.objectMode) ret = state.buffer.shift();else if (!n || n >= state.length) { - // read it all, truncate the list - if (state.decoder) ret = state.buffer.join('');else if (state.buffer.length === 1) ret = state.buffer.head.data;else ret = state.buffer.concat(state.length); - state.buffer.clear(); - } else { - // read part of list - ret = fromListPartial(n, state.buffer, state.decoder); - } - - return ret; - } - - // Extracts only enough buffered data to satisfy the amount requested. - // This function is designed to be inlinable, so please take care when making - // changes to the function body. - function fromListPartial(n, list, hasStrings) { - var ret; - if (n < list.head.data.length) { - // slice is the same for buffers and strings - ret = list.head.data.slice(0, n); - list.head.data = list.head.data.slice(n); - } else if (n === list.head.data.length) { - // first chunk is a perfect match - ret = list.shift(); - } else { - // result spans more than one buffer - ret = hasStrings ? copyFromBufferString(n, list) : copyFromBuffer(n, list); - } - return ret; - } - - // Copies a specified amount of characters from the list of buffered data - // chunks. - // This function is designed to be inlinable, so please take care when making - // changes to the function body. - function copyFromBufferString(n, list) { - var p = list.head; - var c = 1; - var ret = p.data; - n -= ret.length; - while (p = p.next) { - var str = p.data; - var nb = n > str.length ? str.length : n; - if (nb === str.length) ret += str;else ret += str.slice(0, n); - n -= nb; - if (n === 0) { - if (nb === str.length) { - ++c; - if (p.next) list.head = p.next;else list.head = list.tail = null; - } else { - list.head = p; - p.data = str.slice(nb); - } - break; - } - ++c; - } - list.length -= c; - return ret; - } - - // Copies a specified amount of bytes from the list of buffered data chunks. - // This function is designed to be inlinable, so please take care when making - // changes to the function body. - function copyFromBuffer(n, list) { - var ret = Buffer.allocUnsafe(n); - var p = list.head; - var c = 1; - p.data.copy(ret); - n -= p.data.length; - while (p = p.next) { - var buf = p.data; - var nb = n > buf.length ? buf.length : n; - buf.copy(ret, ret.length - n, 0, nb); - n -= nb; - if (n === 0) { - if (nb === buf.length) { - ++c; - if (p.next) list.head = p.next;else list.head = list.tail = null; - } else { - list.head = p; - p.data = buf.slice(nb); - } - break; - } - ++c; - } - list.length -= c; - return ret; - } - - function endReadable(stream) { - var state = stream._readableState; - - // If we get here before consuming all the bytes, then that is a - // bug in node. Should never happen. - if (state.length > 0) throw new Error('"endReadable()" called on non-empty stream'); - - if (!state.endEmitted) { - state.ended = true; - nextTick(endReadableNT, state, stream); - } - } - - function endReadableNT(state, stream) { - // Check that we didn't get one last unshift. - if (!state.endEmitted && state.length === 0) { - state.endEmitted = true; - stream.readable = false; - stream.emit('end'); - } - } - - function forEach(xs, f) { - for (var i = 0, l = xs.length; i < l; i++) { - f(xs[i], i); - } - } - - function indexOf(xs, x) { - for (var i = 0, l = xs.length; i < l; i++) { - if (xs[i] === x) return i; - } - return -1; - } - - // A bit simpler than readable streams. - Writable.WritableState = WritableState; - inherits$1(Writable, EventEmitter); - - function nop() {} - - function WriteReq(chunk, encoding, cb) { - this.chunk = chunk; - this.encoding = encoding; - this.callback = cb; - this.next = null; - } - - function WritableState(options, stream) { - Object.defineProperty(this, 'buffer', { - get: deprecate(function () { - return this.getBuffer(); - }, '_writableState.buffer is deprecated. Use _writableState.getBuffer ' + 'instead.') - }); - options = options || {}; - - // object stream flag to indicate whether or not this stream - // contains buffers or objects. - this.objectMode = !!options.objectMode; - - if (stream instanceof Duplex) this.objectMode = this.objectMode || !!options.writableObjectMode; - - // the point at which write() starts returning false - // Note: 0 is a valid value, means that we always return false if - // the entire buffer is not flushed immediately on write() - var hwm = options.highWaterMark; - var defaultHwm = this.objectMode ? 16 : 16 * 1024; - this.highWaterMark = hwm || hwm === 0 ? hwm : defaultHwm; - - // cast to ints. - this.highWaterMark = ~ ~this.highWaterMark; - - this.needDrain = false; - // at the start of calling end() - this.ending = false; - // when end() has been called, and returned - this.ended = false; - // when 'finish' is emitted - this.finished = false; - - // should we decode strings into buffers before passing to _write? - // this is here so that some node-core streams can optimize string - // handling at a lower level. - var noDecode = options.decodeStrings === false; - this.decodeStrings = !noDecode; - - // Crypto is kind of old and crusty. Historically, its default string - // encoding is 'binary' so we have to make this configurable. - // Everything else in the universe uses 'utf8', though. - this.defaultEncoding = options.defaultEncoding || 'utf8'; - - // not an actual buffer we keep track of, but a measurement - // of how much we're waiting to get pushed to some underlying - // socket or file. - this.length = 0; - - // a flag to see when we're in the middle of a write. - this.writing = false; - - // when true all writes will be buffered until .uncork() call - this.corked = 0; - - // a flag to be able to tell if the onwrite cb is called immediately, - // or on a later tick. We set this to true at first, because any - // actions that shouldn't happen until "later" should generally also - // not happen before the first write call. - this.sync = true; - - // a flag to know if we're processing previously buffered items, which - // may call the _write() callback in the same tick, so that we don't - // end up in an overlapped onwrite situation. - this.bufferProcessing = false; - - // the callback that's passed to _write(chunk,cb) - this.onwrite = function (er) { - onwrite(stream, er); - }; - - // the callback that the user supplies to write(chunk,encoding,cb) - this.writecb = null; - - // the amount that is being written when _write is called. - this.writelen = 0; - - this.bufferedRequest = null; - this.lastBufferedRequest = null; - - // number of pending user-supplied write callbacks - // this must be 0 before 'finish' can be emitted - this.pendingcb = 0; - - // emit prefinish if the only thing we're waiting for is _write cbs - // This is relevant for synchronous Transform streams - this.prefinished = false; - - // True if the error was already emitted and should not be thrown again - this.errorEmitted = false; - - // count buffered requests - this.bufferedRequestCount = 0; - - // allocate the first CorkedRequest, there is always - // one allocated and free to use, and we maintain at most two - this.corkedRequestsFree = new CorkedRequest(this); - } - - WritableState.prototype.getBuffer = function writableStateGetBuffer() { - var current = this.bufferedRequest; - var out = []; - while (current) { - out.push(current); - current = current.next; - } - return out; - }; - function Writable(options) { - - // Writable ctor is applied to Duplexes, though they're not - // instanceof Writable, they're instanceof Readable. - if (!(this instanceof Writable) && !(this instanceof Duplex)) return new Writable(options); - - this._writableState = new WritableState(options, this); - - // legacy. - this.writable = true; - - if (options) { - if (typeof options.write === 'function') this._write = options.write; - - if (typeof options.writev === 'function') this._writev = options.writev; - } - - EventEmitter.call(this); - } - - // Otherwise people can pipe Writable streams, which is just wrong. - Writable.prototype.pipe = function () { - this.emit('error', new Error('Cannot pipe, not readable')); - }; - - function writeAfterEnd(stream, cb) { - var er = new Error('write after end'); - // TODO: defer error events consistently everywhere, not just the cb - stream.emit('error', er); - nextTick(cb, er); - } - - // If we get something that is not a buffer, string, null, or undefined, - // and we're not in objectMode, then that's an error. - // Otherwise stream chunks are all considered to be of length=1, and the - // watermarks determine how many objects to keep in the buffer, rather than - // how many bytes or characters. - function validChunk(stream, state, chunk, cb) { - var valid = true; - var er = false; - // Always throw error if a null is written - // if we are not in object mode then throw - // if it is not a buffer, string, or undefined. - if (chunk === null) { - er = new TypeError('May not write null values to stream'); - } else if (!Buffer.isBuffer(chunk) && typeof chunk !== 'string' && chunk !== undefined && !state.objectMode) { - er = new TypeError('Invalid non-string/buffer chunk'); - } - if (er) { - stream.emit('error', er); - nextTick(cb, er); - valid = false; - } - return valid; - } - - Writable.prototype.write = function (chunk, encoding, cb) { - var state = this._writableState; - var ret = false; - - if (typeof encoding === 'function') { - cb = encoding; - encoding = null; - } - - if (Buffer.isBuffer(chunk)) encoding = 'buffer';else if (!encoding) encoding = state.defaultEncoding; - - if (typeof cb !== 'function') cb = nop; - - if (state.ended) writeAfterEnd(this, cb);else if (validChunk(this, state, chunk, cb)) { - state.pendingcb++; - ret = writeOrBuffer(this, state, chunk, encoding, cb); - } - - return ret; - }; - - Writable.prototype.cork = function () { - var state = this._writableState; - - state.corked++; - }; - - Writable.prototype.uncork = function () { - var state = this._writableState; - - if (state.corked) { - state.corked--; - - if (!state.writing && !state.corked && !state.finished && !state.bufferProcessing && state.bufferedRequest) clearBuffer(this, state); - } - }; - - Writable.prototype.setDefaultEncoding = function setDefaultEncoding(encoding) { - // node::ParseEncoding() requires lower case. - if (typeof encoding === 'string') encoding = encoding.toLowerCase(); - if (!(['hex', 'utf8', 'utf-8', 'ascii', 'binary', 'base64', 'ucs2', 'ucs-2', 'utf16le', 'utf-16le', 'raw'].indexOf((encoding + '').toLowerCase()) > -1)) throw new TypeError('Unknown encoding: ' + encoding); - this._writableState.defaultEncoding = encoding; - return this; - }; - - function decodeChunk(state, chunk, encoding) { - if (!state.objectMode && state.decodeStrings !== false && typeof chunk === 'string') { - chunk = Buffer.from(chunk, encoding); - } - return chunk; - } - - // if we're already writing something, then just put this - // in the queue, and wait our turn. Otherwise, call _write - // If we return false, then we need a drain event, so set that flag. - function writeOrBuffer(stream, state, chunk, encoding, cb) { - chunk = decodeChunk(state, chunk, encoding); - - if (Buffer.isBuffer(chunk)) encoding = 'buffer'; - var len = state.objectMode ? 1 : chunk.length; - - state.length += len; - - var ret = state.length < state.highWaterMark; - // we must ensure that previous needDrain will not be reset to false. - if (!ret) state.needDrain = true; - - if (state.writing || state.corked) { - var last = state.lastBufferedRequest; - state.lastBufferedRequest = new WriteReq(chunk, encoding, cb); - if (last) { - last.next = state.lastBufferedRequest; - } else { - state.bufferedRequest = state.lastBufferedRequest; - } - state.bufferedRequestCount += 1; - } else { - doWrite(stream, state, false, len, chunk, encoding, cb); - } - - return ret; - } - - function doWrite(stream, state, writev, len, chunk, encoding, cb) { - state.writelen = len; - state.writecb = cb; - state.writing = true; - state.sync = true; - if (writev) stream._writev(chunk, state.onwrite);else stream._write(chunk, encoding, state.onwrite); - state.sync = false; - } - - function onwriteError(stream, state, sync, er, cb) { - --state.pendingcb; - if (sync) nextTick(cb, er);else cb(er); - - stream._writableState.errorEmitted = true; - stream.emit('error', er); - } - - function onwriteStateUpdate(state) { - state.writing = false; - state.writecb = null; - state.length -= state.writelen; - state.writelen = 0; - } - - function onwrite(stream, er) { - var state = stream._writableState; - var sync = state.sync; - var cb = state.writecb; - - onwriteStateUpdate(state); - - if (er) onwriteError(stream, state, sync, er, cb);else { - // Check if we're actually ready to finish, but don't emit yet - var finished = needFinish(state); - - if (!finished && !state.corked && !state.bufferProcessing && state.bufferedRequest) { - clearBuffer(stream, state); - } - - if (sync) { - /**/ - nextTick(afterWrite, stream, state, finished, cb); - /**/ - } else { - afterWrite(stream, state, finished, cb); - } - } - } - - function afterWrite(stream, state, finished, cb) { - if (!finished) onwriteDrain(stream, state); - state.pendingcb--; - cb(); - finishMaybe(stream, state); - } - - // Must force callback to be called on nextTick, so that we don't - // emit 'drain' before the write() consumer gets the 'false' return - // value, and has a chance to attach a 'drain' listener. - function onwriteDrain(stream, state) { - if (state.length === 0 && state.needDrain) { - state.needDrain = false; - stream.emit('drain'); - } - } - - // if there's something in the buffer waiting, then process it - function clearBuffer(stream, state) { - state.bufferProcessing = true; - var entry = state.bufferedRequest; - - if (stream._writev && entry && entry.next) { - // Fast case, write everything using _writev() - var l = state.bufferedRequestCount; - var buffer = new Array(l); - var holder = state.corkedRequestsFree; - holder.entry = entry; - - var count = 0; - while (entry) { - buffer[count] = entry; - entry = entry.next; - count += 1; - } - - doWrite(stream, state, true, state.length, buffer, '', holder.finish); - - // doWrite is almost always async, defer these to save a bit of time - // as the hot path ends with doWrite - state.pendingcb++; - state.lastBufferedRequest = null; - if (holder.next) { - state.corkedRequestsFree = holder.next; - holder.next = null; - } else { - state.corkedRequestsFree = new CorkedRequest(state); - } - } else { - // Slow case, write chunks one-by-one - while (entry) { - var chunk = entry.chunk; - var encoding = entry.encoding; - var cb = entry.callback; - var len = state.objectMode ? 1 : chunk.length; - - doWrite(stream, state, false, len, chunk, encoding, cb); - entry = entry.next; - // if we didn't call the onwrite immediately, then - // it means that we need to wait until it does. - // also, that means that the chunk and cb are currently - // being processed, so move the buffer counter past them. - if (state.writing) { - break; - } - } - - if (entry === null) state.lastBufferedRequest = null; - } - - state.bufferedRequestCount = 0; - state.bufferedRequest = entry; - state.bufferProcessing = false; - } - - Writable.prototype._write = function (chunk, encoding, cb) { - cb(new Error('not implemented')); - }; - - Writable.prototype._writev = null; - - Writable.prototype.end = function (chunk, encoding, cb) { - var state = this._writableState; - - if (typeof chunk === 'function') { - cb = chunk; - chunk = null; - encoding = null; - } else if (typeof encoding === 'function') { - cb = encoding; - encoding = null; - } - - if (chunk !== null && chunk !== undefined) this.write(chunk, encoding); - - // .end() fully uncorks - if (state.corked) { - state.corked = 1; - this.uncork(); - } - - // ignore unnecessary end() calls. - if (!state.ending && !state.finished) endWritable(this, state, cb); - }; - - function needFinish(state) { - return state.ending && state.length === 0 && state.bufferedRequest === null && !state.finished && !state.writing; - } - - function prefinish(stream, state) { - if (!state.prefinished) { - state.prefinished = true; - stream.emit('prefinish'); - } - } - - function finishMaybe(stream, state) { - var need = needFinish(state); - if (need) { - if (state.pendingcb === 0) { - prefinish(stream, state); - state.finished = true; - stream.emit('finish'); - } else { - prefinish(stream, state); - } - } - return need; - } - - function endWritable(stream, state, cb) { - state.ending = true; - finishMaybe(stream, state); - if (cb) { - if (state.finished) nextTick(cb);else stream.once('finish', cb); - } - state.ended = true; - stream.writable = false; - } - - // It seems a linked list but it is not - // there will be only 2 of these for each stream - function CorkedRequest(state) { - var _this = this; - - this.next = null; - this.entry = null; - - this.finish = function (err) { - var entry = _this.entry; - _this.entry = null; - while (entry) { - var cb = entry.callback; - state.pendingcb--; - cb(err); - entry = entry.next; - } - if (state.corkedRequestsFree) { - state.corkedRequestsFree.next = _this; - } else { - state.corkedRequestsFree = _this; - } - }; - } - - inherits$1(Duplex, Readable); - - var keys = Object.keys(Writable.prototype); - for (var v = 0; v < keys.length; v++) { - var method = keys[v]; - if (!Duplex.prototype[method]) Duplex.prototype[method] = Writable.prototype[method]; - } - function Duplex(options) { - if (!(this instanceof Duplex)) return new Duplex(options); - - Readable.call(this, options); - Writable.call(this, options); - - if (options && options.readable === false) this.readable = false; - - if (options && options.writable === false) this.writable = false; - - this.allowHalfOpen = true; - if (options && options.allowHalfOpen === false) this.allowHalfOpen = false; - - this.once('end', onend); - } - - // the no-half-open enforcer - function onend() { - // if we allow half-open state, or if the writable side ended, - // then we're ok. - if (this.allowHalfOpen || this._writableState.ended) return; - - // no more data can be written. - // But allow more writes to happen in this tick. - nextTick(onEndNT, this); - } - - function onEndNT(self) { - self.end(); - } - - // a transform stream is a readable/writable stream where you do - inherits$1(Transform, Duplex); - - function TransformState(stream) { - this.afterTransform = function (er, data) { - return afterTransform(stream, er, data); - }; - - this.needTransform = false; - this.transforming = false; - this.writecb = null; - this.writechunk = null; - this.writeencoding = null; - } - - function afterTransform(stream, er, data) { - var ts = stream._transformState; - ts.transforming = false; - - var cb = ts.writecb; - - if (!cb) return stream.emit('error', new Error('no writecb in Transform class')); - - ts.writechunk = null; - ts.writecb = null; - - if (data !== null && data !== undefined) stream.push(data); - - cb(er); - - var rs = stream._readableState; - rs.reading = false; - if (rs.needReadable || rs.length < rs.highWaterMark) { - stream._read(rs.highWaterMark); - } - } - function Transform(options) { - if (!(this instanceof Transform)) return new Transform(options); - - Duplex.call(this, options); - - this._transformState = new TransformState(this); - - // when the writable side finishes, then flush out anything remaining. - var stream = this; - - // start out asking for a readable event once data is transformed. - this._readableState.needReadable = true; - - // we have implemented the _read method, and done the other things - // that Readable wants before the first _read call, so unset the - // sync guard flag. - this._readableState.sync = false; - - if (options) { - if (typeof options.transform === 'function') this._transform = options.transform; - - if (typeof options.flush === 'function') this._flush = options.flush; - } - - this.once('prefinish', function () { - if (typeof this._flush === 'function') this._flush(function (er) { - done(stream, er); - });else done(stream); - }); - } - - Transform.prototype.push = function (chunk, encoding) { - this._transformState.needTransform = false; - return Duplex.prototype.push.call(this, chunk, encoding); - }; - - // This is the part where you do stuff! - // override this function in implementation classes. - // 'chunk' is an input chunk. - // - // Call `push(newChunk)` to pass along transformed output - // to the readable side. You may call 'push' zero or more times. - // - // Call `cb(err)` when you are done with this chunk. If you pass - // an error, then that'll put the hurt on the whole operation. If you - // never call cb(), then you'll never get another chunk. - Transform.prototype._transform = function (chunk, encoding, cb) { - throw new Error('Not implemented'); - }; - - Transform.prototype._write = function (chunk, encoding, cb) { - var ts = this._transformState; - ts.writecb = cb; - ts.writechunk = chunk; - ts.writeencoding = encoding; - if (!ts.transforming) { - var rs = this._readableState; - if (ts.needTransform || rs.needReadable || rs.length < rs.highWaterMark) this._read(rs.highWaterMark); - } - }; - - // Doesn't matter what the args are here. - // _transform does all the work. - // That we got here means that the readable side wants more data. - Transform.prototype._read = function (n) { - var ts = this._transformState; - - if (ts.writechunk !== null && ts.writecb && !ts.transforming) { - ts.transforming = true; - this._transform(ts.writechunk, ts.writeencoding, ts.afterTransform); - } else { - // mark that we need a transform, so that any data that comes in - // will get processed, now that we've asked for it. - ts.needTransform = true; + class CsvError extends Error { + constructor(code, message, options, ...contexts) { + if(Array.isArray(message)) message = message.join(' '); + super(message); + if(Error.captureStackTrace !== undefined){ + Error.captureStackTrace(this, CsvError); + } + this.code = code; + for(const context of contexts){ + for(const key in context){ + const value = context[key]; + this[key] = isBuffer(value) ? value.toString(options.encoding) : value == null ? value : JSON.parse(JSON.stringify(value)); + } + } } - }; - - function done(stream, er) { - if (er) return stream.emit('error', er); - - // if there's nothing in the write buffer, then that means - // that nothing more will ever be provided - var ws = stream._writableState; - var ts = stream._transformState; - - if (ws.length) throw new Error('Calling transform done when ws.length != 0'); - - if (ts.transforming) throw new Error('Calling transform done when still transforming'); - - return stream.push(null); } - inherits$1(PassThrough, Transform); - function PassThrough(options) { - if (!(this instanceof PassThrough)) return new PassThrough(options); - - Transform.call(this, options); - } - - PassThrough.prototype._transform = function (chunk, encoding, cb) { - cb(null, chunk); + const is_object = function(obj){ + return (typeof obj === 'object' && obj !== null && !Array.isArray(obj)); }; - inherits$1(Stream, EventEmitter); - Stream.Readable = Readable; - Stream.Writable = Writable; - Stream.Duplex = Duplex; - Stream.Transform = Transform; - Stream.PassThrough = PassThrough; - - // Backwards-compat with node 0.4.x - Stream.Stream = Stream; - - // old-style streams. Note that the pipe method (the only relevant - // part of this class) is overridden in the Readable class. - - function Stream() { - EventEmitter.call(this); - } - - Stream.prototype.pipe = function(dest, options) { - var source = this; - - function ondata(chunk) { - if (dest.writable) { - if (false === dest.write(chunk) && source.pause) { - source.pause(); + const normalize_columns_array = function(columns){ + const normalizedColumns = []; + for(let i = 0, l = columns.length; i < l; i++){ + const column = columns[i]; + if(column === undefined || column === null || column === false){ + normalizedColumns[i] = { disabled: true }; + }else if(typeof column === 'string'){ + normalizedColumns[i] = { name: column }; + }else if(is_object(column)){ + if(typeof column.name !== 'string'){ + throw new CsvError('CSV_OPTION_COLUMNS_MISSING_NAME', [ + 'Option columns missing name:', + `property "name" is required at position ${i}`, + 'when column is an object literal' + ]); } + normalizedColumns[i] = column; + }else { + throw new CsvError('CSV_INVALID_COLUMN_DEFINITION', [ + 'Invalid column definition:', + 'expect a string or a literal object,', + `got ${JSON.stringify(column)} at position ${i}` + ]); } } - - source.on('data', ondata); - - function ondrain() { - if (source.readable && source.resume) { - source.resume(); - } - } - - dest.on('drain', ondrain); - - // If the 'end' option is not supplied, dest.end() will be called when - // source gets the 'end' or 'close' events. Only dest.end() once. - if (!dest._isStdio && (!options || options.end !== false)) { - source.on('end', onend); - source.on('close', onclose); - } - - var didOnEnd = false; - function onend() { - if (didOnEnd) return; - didOnEnd = true; - - dest.end(); - } - - - function onclose() { - if (didOnEnd) return; - didOnEnd = true; - - if (typeof dest.destroy === 'function') dest.destroy(); - } - - // don't leave dangling pipes when there are errors. - function onerror(er) { - cleanup(); - if (EventEmitter.listenerCount(this, 'error') === 0) { - throw er; // Unhandled stream error in pipe. - } - } - - source.on('error', onerror); - dest.on('error', onerror); - - // remove all the event listeners that were added. - function cleanup() { - source.removeListener('data', ondata); - dest.removeListener('drain', ondrain); - - source.removeListener('end', onend); - source.removeListener('close', onclose); - - source.removeListener('error', onerror); - dest.removeListener('error', onerror); - - source.removeListener('end', cleanup); - source.removeListener('close', cleanup); - - dest.removeListener('close', cleanup); - } - - source.on('end', cleanup); - source.on('close', cleanup); - - dest.on('close', cleanup); - - dest.emit('pipe', source); - - // Allow for unix-like usage: A.pipe(B).pipe(C) - return dest; + return normalizedColumns; }; class ResizeableBuffer{ @@ -5034,1232 +2080,1171 @@ } } - // white space characters - // https://en.wikipedia.org/wiki/Whitespace_character - // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions/Character_Classes#Types - // \f\n\r\t\v\u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff - const tab = 9; - const nl = 10; // \n, 0x0A in hexadecimal, 10 in decimal - const np = 12; - const cr = 13; // \r, 0x0D in hexadécimal, 13 in decimal - const space = 32; - const boms = { - // Note, the following are equals: - // Buffer.from("\ufeff") - // Buffer.from([239, 187, 191]) - // Buffer.from('EFBBBF', 'hex') - 'utf8': Buffer.from([239, 187, 191]), - // Note, the following are equals: - // Buffer.from "\ufeff", 'utf16le - // Buffer.from([255, 254]) - 'utf16le': Buffer.from([255, 254]) + const init_state = function(options){ + return { + bomSkipped: false, + bufBytesStart: 0, + castField: options.cast_function, + commenting: false, + // Current error encountered by a record + error: undefined, + enabled: options.from_line === 1, + escaping: false, + escapeIsQuote: isBuffer(options.escape) && isBuffer(options.quote) && Buffer.compare(options.escape, options.quote) === 0, + // columns can be `false`, `true`, `Array` + expectedRecordLength: Array.isArray(options.columns) ? options.columns.length : undefined, + field: new ResizeableBuffer(20), + firstLineToHeaders: options.cast_first_line_to_header, + needMoreDataSize: Math.max( + // Skip if the remaining buffer smaller than comment + options.comment !== null ? options.comment.length : 0, + // Skip if the remaining buffer can be delimiter + ...options.delimiter.map((delimiter) => delimiter.length), + // Skip if the remaining buffer can be escape sequence + options.quote !== null ? options.quote.length : 0, + ), + previousBuf: undefined, + quoting: false, + stop: false, + rawBuffer: new ResizeableBuffer(100), + record: [], + recordHasError: false, + record_length: 0, + recordDelimiterMaxLength: options.record_delimiter.length === 0 ? 2 : Math.max(...options.record_delimiter.map((v) => v.length)), + trimChars: [Buffer.from(' ', options.encoding)[0], Buffer.from('\t', options.encoding)[0]], + wasQuoting: false, + wasRowDelimiter: false + }; }; - class CsvError extends Error { - constructor(code, message, options, ...contexts) { - if(Array.isArray(message)) message = message.join(' '); - super(message); - if(Error.captureStackTrace !== undefined){ - Error.captureStackTrace(this, CsvError); - } - this.code = code; - for(const context of contexts){ - for(const key in context){ - const value = context[key]; - this[key] = isBuffer(value) ? value.toString(options.encoding) : value == null ? value : JSON.parse(JSON.stringify(value)); - } - } - } - } - const underscore = function(str){ return str.replace(/([A-Z])/g, function(_, match){ return '_' + match.toLowerCase(); }); }; - const isObject = function(obj){ - return (typeof obj === 'object' && obj !== null && !Array.isArray(obj)); - }; - - const isRecordEmpty = function(record){ - return record.every((field) => field == null || field.toString && field.toString().trim() === ''); - }; - - const normalizeColumnsArray = function(columns){ - const normalizedColumns = []; - for(let i = 0, l = columns.length; i < l; i++){ - const column = columns[i]; - if(column === undefined || column === null || column === false){ - normalizedColumns[i] = { disabled: true }; - }else if(typeof column === 'string'){ - normalizedColumns[i] = { name: column }; - }else if(isObject(column)){ - if(typeof column.name !== 'string'){ - throw new CsvError('CSV_OPTION_COLUMNS_MISSING_NAME', [ - 'Option columns missing name:', - `property "name" is required at position ${i}`, - 'when column is an object literal' - ]); - } - normalizedColumns[i] = column; - }else { - throw new CsvError('CSV_INVALID_COLUMN_DEFINITION', [ - 'Invalid column definition:', - 'expect a string or a literal object,', - `got ${JSON.stringify(column)} at position ${i}` - ]); - } - } - return normalizedColumns; - }; - - class Parser extends Transform { - constructor(opts = {}){ - super({...{readableObjectMode: true}, ...opts, encoding: null}); - this.__originalOptions = opts; - this.__normalizeOptions(opts); - } - __normalizeOptions(opts){ - const options = {}; - // Merge with user options - for(const opt in opts){ - options[underscore(opt)] = opts[opt]; - } - // Normalize option `encoding` - // Note: defined first because other options depends on it - // to convert chars/strings into buffers. - if(options.encoding === undefined || options.encoding === true){ - options.encoding = 'utf8'; - }else if(options.encoding === null || options.encoding === false){ - options.encoding = null; - }else if(typeof options.encoding !== 'string' && options.encoding !== null){ - throw new CsvError('CSV_INVALID_OPTION_ENCODING', [ - 'Invalid option encoding:', - 'encoding must be a string or null to return a buffer,', - `got ${JSON.stringify(options.encoding)}` - ], options); - } - // Normalize option `bom` - if(options.bom === undefined || options.bom === null || options.bom === false){ - options.bom = false; - }else if(options.bom !== true){ - throw new CsvError('CSV_INVALID_OPTION_BOM', [ - 'Invalid option bom:', 'bom must be true,', - `got ${JSON.stringify(options.bom)}` - ], options); - } - // Normalize option `cast` - let fnCastField = null; - if(options.cast === undefined || options.cast === null || options.cast === false || options.cast === ''){ - options.cast = undefined; - }else if(typeof options.cast === 'function'){ - fnCastField = options.cast; - options.cast = true; - }else if(options.cast !== true){ - throw new CsvError('CSV_INVALID_OPTION_CAST', [ - 'Invalid option cast:', 'cast must be true or a function,', - `got ${JSON.stringify(options.cast)}` - ], options); - } - // Normalize option `cast_date` - if(options.cast_date === undefined || options.cast_date === null || options.cast_date === false || options.cast_date === ''){ - options.cast_date = false; - }else if(options.cast_date === true){ - options.cast_date = function(value){ - const date = Date.parse(value); - return !isNaN(date) ? new Date(date) : value; - }; - }else { - throw new CsvError('CSV_INVALID_OPTION_CAST_DATE', [ - 'Invalid option cast_date:', 'cast_date must be true or a function,', - `got ${JSON.stringify(options.cast_date)}` - ], options); - } - // Normalize option `columns` - let fnFirstLineToHeaders = null; - if(options.columns === true){ - // Fields in the first line are converted as-is to columns - fnFirstLineToHeaders = undefined; - }else if(typeof options.columns === 'function'){ - fnFirstLineToHeaders = options.columns; - options.columns = true; - }else if(Array.isArray(options.columns)){ - options.columns = normalizeColumnsArray(options.columns); - }else if(options.columns === undefined || options.columns === null || options.columns === false){ - options.columns = false; - }else { - throw new CsvError('CSV_INVALID_OPTION_COLUMNS', [ - 'Invalid option columns:', - 'expect an array, a function or true,', - `got ${JSON.stringify(options.columns)}` + const normalize_options = function(opts){ + const options = {}; + // Merge with user options + for(const opt in opts){ + options[underscore(opt)] = opts[opt]; + } + // Normalize option `encoding` + // Note: defined first because other options depends on it + // to convert chars/strings into buffers. + if(options.encoding === undefined || options.encoding === true){ + options.encoding = 'utf8'; + }else if(options.encoding === null || options.encoding === false){ + options.encoding = null; + }else if(typeof options.encoding !== 'string' && options.encoding !== null){ + throw new CsvError('CSV_INVALID_OPTION_ENCODING', [ + 'Invalid option encoding:', + 'encoding must be a string or null to return a buffer,', + `got ${JSON.stringify(options.encoding)}` + ], options); + } + // Normalize option `bom` + if(options.bom === undefined || options.bom === null || options.bom === false){ + options.bom = false; + }else if(options.bom !== true){ + throw new CsvError('CSV_INVALID_OPTION_BOM', [ + 'Invalid option bom:', 'bom must be true,', + `got ${JSON.stringify(options.bom)}` + ], options); + } + // Normalize option `cast` + options.cast_function = null; + if(options.cast === undefined || options.cast === null || options.cast === false || options.cast === ''){ + options.cast = undefined; + }else if(typeof options.cast === 'function'){ + options.cast_function = options.cast; + options.cast = true; + }else if(options.cast !== true){ + throw new CsvError('CSV_INVALID_OPTION_CAST', [ + 'Invalid option cast:', 'cast must be true or a function,', + `got ${JSON.stringify(options.cast)}` + ], options); + } + // Normalize option `cast_date` + if(options.cast_date === undefined || options.cast_date === null || options.cast_date === false || options.cast_date === ''){ + options.cast_date = false; + }else if(options.cast_date === true){ + options.cast_date = function(value){ + const date = Date.parse(value); + return !isNaN(date) ? new Date(date) : value; + }; + }else { + throw new CsvError('CSV_INVALID_OPTION_CAST_DATE', [ + 'Invalid option cast_date:', 'cast_date must be true or a function,', + `got ${JSON.stringify(options.cast_date)}` + ], options); + } + // Normalize option `columns` + options.cast_first_line_to_header = null; + if(options.columns === true){ + // Fields in the first line are converted as-is to columns + options.cast_first_line_to_header = undefined; + }else if(typeof options.columns === 'function'){ + options.cast_first_line_to_header = options.columns; + options.columns = true; + }else if(Array.isArray(options.columns)){ + options.columns = normalize_columns_array(options.columns); + }else if(options.columns === undefined || options.columns === null || options.columns === false){ + options.columns = false; + }else { + throw new CsvError('CSV_INVALID_OPTION_COLUMNS', [ + 'Invalid option columns:', + 'expect an array, a function or true,', + `got ${JSON.stringify(options.columns)}` + ], options); + } + // Normalize option `group_columns_by_name` + if(options.group_columns_by_name === undefined || options.group_columns_by_name === null || options.group_columns_by_name === false){ + options.group_columns_by_name = false; + }else if(options.group_columns_by_name !== true){ + throw new CsvError('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ + 'Invalid option group_columns_by_name:', + 'expect an boolean,', + `got ${JSON.stringify(options.group_columns_by_name)}` + ], options); + }else if(options.columns === false){ + throw new CsvError('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ + 'Invalid option group_columns_by_name:', + 'the `columns` mode must be activated.' + ], options); + } + // Normalize option `comment` + if(options.comment === undefined || options.comment === null || options.comment === false || options.comment === ''){ + options.comment = null; + }else { + if(typeof options.comment === 'string'){ + options.comment = Buffer.from(options.comment, options.encoding); + } + if(!isBuffer(options.comment)){ + throw new CsvError('CSV_INVALID_OPTION_COMMENT', [ + 'Invalid option comment:', + 'comment must be a buffer or a string,', + `got ${JSON.stringify(options.comment)}` ], options); } - // Normalize option `group_columns_by_name` - if(options.group_columns_by_name === undefined || options.group_columns_by_name === null || options.group_columns_by_name === false){ - options.group_columns_by_name = false; - }else if(options.group_columns_by_name !== true){ - throw new CsvError('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ - 'Invalid option group_columns_by_name:', - 'expect an boolean,', - `got ${JSON.stringify(options.group_columns_by_name)}` - ], options); - }else if(options.columns === false){ - throw new CsvError('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ - 'Invalid option group_columns_by_name:', - 'the `columns` mode must be activated.' - ], options); + } + // Normalize option `delimiter` + const delimiter_json = JSON.stringify(options.delimiter); + if(!Array.isArray(options.delimiter)) options.delimiter = [options.delimiter]; + if(options.delimiter.length === 0){ + throw new CsvError('CSV_INVALID_OPTION_DELIMITER', [ + 'Invalid option delimiter:', + 'delimiter must be a non empty string or buffer or array of string|buffer,', + `got ${delimiter_json}` + ], options); + } + options.delimiter = options.delimiter.map(function(delimiter){ + if(delimiter === undefined || delimiter === null || delimiter === false){ + return Buffer.from(',', options.encoding); } - // Normalize option `comment` - if(options.comment === undefined || options.comment === null || options.comment === false || options.comment === ''){ - options.comment = null; - }else { - if(typeof options.comment === 'string'){ - options.comment = Buffer.from(options.comment, options.encoding); - } - if(!isBuffer(options.comment)){ - throw new CsvError('CSV_INVALID_OPTION_COMMENT', [ - 'Invalid option comment:', - 'comment must be a buffer or a string,', - `got ${JSON.stringify(options.comment)}` - ], options); - } + if(typeof delimiter === 'string'){ + delimiter = Buffer.from(delimiter, options.encoding); } - // Normalize option `delimiter` - const delimiter_json = JSON.stringify(options.delimiter); - if(!Array.isArray(options.delimiter)) options.delimiter = [options.delimiter]; - if(options.delimiter.length === 0){ + if(!isBuffer(delimiter) || delimiter.length === 0){ throw new CsvError('CSV_INVALID_OPTION_DELIMITER', [ 'Invalid option delimiter:', 'delimiter must be a non empty string or buffer or array of string|buffer,', `got ${delimiter_json}` ], options); } - options.delimiter = options.delimiter.map(function(delimiter){ - if(delimiter === undefined || delimiter === null || delimiter === false){ - return Buffer.from(',', options.encoding); - } - if(typeof delimiter === 'string'){ - delimiter = Buffer.from(delimiter, options.encoding); - } - if(!isBuffer(delimiter) || delimiter.length === 0){ - throw new CsvError('CSV_INVALID_OPTION_DELIMITER', [ - 'Invalid option delimiter:', - 'delimiter must be a non empty string or buffer or array of string|buffer,', - `got ${delimiter_json}` - ], options); - } - return delimiter; - }); - // Normalize option `escape` - if(options.escape === undefined || options.escape === true){ - options.escape = Buffer.from('"', options.encoding); - }else if(typeof options.escape === 'string'){ - options.escape = Buffer.from(options.escape, options.encoding); - }else if (options.escape === null || options.escape === false){ - options.escape = null; - } - if(options.escape !== null){ - if(!isBuffer(options.escape)){ - throw new Error(`Invalid Option: escape must be a buffer, a string or a boolean, got ${JSON.stringify(options.escape)}`); - } - } - // Normalize option `from` - if(options.from === undefined || options.from === null){ - options.from = 1; - }else { - if(typeof options.from === 'string' && /\d+/.test(options.from)){ - options.from = parseInt(options.from); - } - if(Number.isInteger(options.from)){ - if(options.from < 0){ - throw new Error(`Invalid Option: from must be a positive integer, got ${JSON.stringify(opts.from)}`); - } - }else { - throw new Error(`Invalid Option: from must be an integer, got ${JSON.stringify(options.from)}`); - } - } - // Normalize option `from_line` - if(options.from_line === undefined || options.from_line === null){ - options.from_line = 1; - }else { - if(typeof options.from_line === 'string' && /\d+/.test(options.from_line)){ - options.from_line = parseInt(options.from_line); - } - if(Number.isInteger(options.from_line)){ - if(options.from_line <= 0){ - throw new Error(`Invalid Option: from_line must be a positive integer greater than 0, got ${JSON.stringify(opts.from_line)}`); - } - }else { - throw new Error(`Invalid Option: from_line must be an integer, got ${JSON.stringify(opts.from_line)}`); - } - } - // Normalize options `ignore_last_delimiters` - if(options.ignore_last_delimiters === undefined || options.ignore_last_delimiters === null){ - options.ignore_last_delimiters = false; - }else if(typeof options.ignore_last_delimiters === 'number'){ - options.ignore_last_delimiters = Math.floor(options.ignore_last_delimiters); - if(options.ignore_last_delimiters === 0){ - options.ignore_last_delimiters = false; - } - }else if(typeof options.ignore_last_delimiters !== 'boolean'){ - throw new CsvError('CSV_INVALID_OPTION_IGNORE_LAST_DELIMITERS', [ - 'Invalid option `ignore_last_delimiters`:', - 'the value must be a boolean value or an integer,', - `got ${JSON.stringify(options.ignore_last_delimiters)}` - ], options); - } - if(options.ignore_last_delimiters === true && options.columns === false){ - throw new CsvError('CSV_IGNORE_LAST_DELIMITERS_REQUIRES_COLUMNS', [ - 'The option `ignore_last_delimiters`', - 'requires the activation of the `columns` option' - ], options); - } - // Normalize option `info` - if(options.info === undefined || options.info === null || options.info === false){ - options.info = false; - }else if(options.info !== true){ - throw new Error(`Invalid Option: info must be true, got ${JSON.stringify(options.info)}`); - } - // Normalize option `max_record_size` - if(options.max_record_size === undefined || options.max_record_size === null || options.max_record_size === false){ - options.max_record_size = 0; - }else if(Number.isInteger(options.max_record_size) && options.max_record_size >= 0);else if(typeof options.max_record_size === 'string' && /\d+/.test(options.max_record_size)){ - options.max_record_size = parseInt(options.max_record_size); - }else { - throw new Error(`Invalid Option: max_record_size must be a positive integer, got ${JSON.stringify(options.max_record_size)}`); - } - // Normalize option `objname` - if(options.objname === undefined || options.objname === null || options.objname === false){ - options.objname = undefined; - }else if(isBuffer(options.objname)){ - if(options.objname.length === 0){ - throw new Error(`Invalid Option: objname must be a non empty buffer`); - } - if(options.encoding === null);else { - options.objname = options.objname.toString(options.encoding); - } - }else if(typeof options.objname === 'string'){ - if(options.objname.length === 0){ - throw new Error(`Invalid Option: objname must be a non empty string`); - } - // Great, nothing to do - }else if(typeof options.objname === 'number');else { - throw new Error(`Invalid Option: objname must be a string or a buffer, got ${options.objname}`); - } - if(options.objname !== undefined){ - if(typeof options.objname === 'number'){ - if(options.columns !== false){ - throw Error('Invalid Option: objname index cannot be combined with columns or be defined as a field'); - } - }else { // A string or a buffer - if(options.columns === false){ - throw Error('Invalid Option: objname field must be combined with columns or be defined as an index'); - } - } - } - // Normalize option `on_record` - if(options.on_record === undefined || options.on_record === null){ - options.on_record = undefined; - }else if(typeof options.on_record !== 'function'){ - throw new CsvError('CSV_INVALID_OPTION_ON_RECORD', [ - 'Invalid option `on_record`:', - 'expect a function,', - `got ${JSON.stringify(options.on_record)}` - ], options); - } - // Normalize option `quote` - if(options.quote === null || options.quote === false || options.quote === ''){ - options.quote = null; - }else { - if(options.quote === undefined || options.quote === true){ - options.quote = Buffer.from('"', options.encoding); - }else if(typeof options.quote === 'string'){ - options.quote = Buffer.from(options.quote, options.encoding); - } - if(!isBuffer(options.quote)){ - throw new Error(`Invalid Option: quote must be a buffer or a string, got ${JSON.stringify(options.quote)}`); - } - } - // Normalize option `raw` - if(options.raw === undefined || options.raw === null || options.raw === false){ - options.raw = false; - }else if(options.raw !== true){ - throw new Error(`Invalid Option: raw must be true, got ${JSON.stringify(options.raw)}`); - } - // Normalize option `record_delimiter` - if(options.record_delimiter === undefined){ - options.record_delimiter = []; - }else if(typeof options.record_delimiter === 'string' || isBuffer(options.record_delimiter)){ - if(options.record_delimiter.length === 0){ - throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ - 'Invalid option `record_delimiter`:', - 'value must be a non empty string or buffer,', - `got ${JSON.stringify(options.record_delimiter)}` - ], options); - } - options.record_delimiter = [options.record_delimiter]; - }else if(!Array.isArray(options.record_delimiter)){ - throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ - 'Invalid option `record_delimiter`:', - 'value must be a string, a buffer or array of string|buffer,', - `got ${JSON.stringify(options.record_delimiter)}` - ], options); - } - options.record_delimiter = options.record_delimiter.map(function(rd, i){ - if(typeof rd !== 'string' && ! isBuffer(rd)){ - throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ - 'Invalid option `record_delimiter`:', - 'value must be a string, a buffer or array of string|buffer', - `at index ${i},`, - `got ${JSON.stringify(rd)}` - ], options); - }else if(rd.length === 0){ - throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ - 'Invalid option `record_delimiter`:', - 'value must be a non empty string or buffer', - `at index ${i},`, - `got ${JSON.stringify(rd)}` - ], options); - } - if(typeof rd === 'string'){ - rd = Buffer.from(rd, options.encoding); + return delimiter; + }); + // Normalize option `escape` + if(options.escape === undefined || options.escape === true){ + options.escape = Buffer.from('"', options.encoding); + }else if(typeof options.escape === 'string'){ + options.escape = Buffer.from(options.escape, options.encoding); + }else if (options.escape === null || options.escape === false){ + options.escape = null; + } + if(options.escape !== null){ + if(!isBuffer(options.escape)){ + throw new Error(`Invalid Option: escape must be a buffer, a string or a boolean, got ${JSON.stringify(options.escape)}`); + } + } + // Normalize option `from` + if(options.from === undefined || options.from === null){ + options.from = 1; + }else { + if(typeof options.from === 'string' && /\d+/.test(options.from)){ + options.from = parseInt(options.from); + } + if(Number.isInteger(options.from)){ + if(options.from < 0){ + throw new Error(`Invalid Option: from must be a positive integer, got ${JSON.stringify(opts.from)}`); } - return rd; - }); - // Normalize option `relax_column_count` - if(typeof options.relax_column_count === 'boolean');else if(options.relax_column_count === undefined || options.relax_column_count === null){ - options.relax_column_count = false; - }else { - throw new Error(`Invalid Option: relax_column_count must be a boolean, got ${JSON.stringify(options.relax_column_count)}`); - } - if(typeof options.relax_column_count_less === 'boolean');else if(options.relax_column_count_less === undefined || options.relax_column_count_less === null){ - options.relax_column_count_less = false; - }else { - throw new Error(`Invalid Option: relax_column_count_less must be a boolean, got ${JSON.stringify(options.relax_column_count_less)}`); - } - if(typeof options.relax_column_count_more === 'boolean');else if(options.relax_column_count_more === undefined || options.relax_column_count_more === null){ - options.relax_column_count_more = false; - }else { - throw new Error(`Invalid Option: relax_column_count_more must be a boolean, got ${JSON.stringify(options.relax_column_count_more)}`); - } - // Normalize option `relax_quotes` - if(typeof options.relax_quotes === 'boolean');else if(options.relax_quotes === undefined || options.relax_quotes === null){ - options.relax_quotes = false; - }else { - throw new Error(`Invalid Option: relax_quotes must be a boolean, got ${JSON.stringify(options.relax_quotes)}`); - } - // Normalize option `skip_empty_lines` - if(typeof options.skip_empty_lines === 'boolean');else if(options.skip_empty_lines === undefined || options.skip_empty_lines === null){ - options.skip_empty_lines = false; }else { - throw new Error(`Invalid Option: skip_empty_lines must be a boolean, got ${JSON.stringify(options.skip_empty_lines)}`); + throw new Error(`Invalid Option: from must be an integer, got ${JSON.stringify(options.from)}`); } - // Normalize option `skip_records_with_empty_values` - if(typeof options.skip_records_with_empty_values === 'boolean');else if(options.skip_records_with_empty_values === undefined || options.skip_records_with_empty_values === null){ - options.skip_records_with_empty_values = false; - }else { - throw new Error(`Invalid Option: skip_records_with_empty_values must be a boolean, got ${JSON.stringify(options.skip_records_with_empty_values)}`); + } + // Normalize option `from_line` + if(options.from_line === undefined || options.from_line === null){ + options.from_line = 1; + }else { + if(typeof options.from_line === 'string' && /\d+/.test(options.from_line)){ + options.from_line = parseInt(options.from_line); } - // Normalize option `skip_records_with_error` - if(typeof options.skip_records_with_error === 'boolean');else if(options.skip_records_with_error === undefined || options.skip_records_with_error === null){ - options.skip_records_with_error = false; + if(Number.isInteger(options.from_line)){ + if(options.from_line <= 0){ + throw new Error(`Invalid Option: from_line must be a positive integer greater than 0, got ${JSON.stringify(opts.from_line)}`); + } }else { - throw new Error(`Invalid Option: skip_records_with_error must be a boolean, got ${JSON.stringify(options.skip_records_with_error)}`); - } - // Normalize option `rtrim` - if(options.rtrim === undefined || options.rtrim === null || options.rtrim === false){ - options.rtrim = false; - }else if(options.rtrim !== true){ - throw new Error(`Invalid Option: rtrim must be a boolean, got ${JSON.stringify(options.rtrim)}`); + throw new Error(`Invalid Option: from_line must be an integer, got ${JSON.stringify(opts.from_line)}`); } - // Normalize option `ltrim` - if(options.ltrim === undefined || options.ltrim === null || options.ltrim === false){ - options.ltrim = false; - }else if(options.ltrim !== true){ - throw new Error(`Invalid Option: ltrim must be a boolean, got ${JSON.stringify(options.ltrim)}`); + } + // Normalize options `ignore_last_delimiters` + if(options.ignore_last_delimiters === undefined || options.ignore_last_delimiters === null){ + options.ignore_last_delimiters = false; + }else if(typeof options.ignore_last_delimiters === 'number'){ + options.ignore_last_delimiters = Math.floor(options.ignore_last_delimiters); + if(options.ignore_last_delimiters === 0){ + options.ignore_last_delimiters = false; } - // Normalize option `trim` - if(options.trim === undefined || options.trim === null || options.trim === false){ - options.trim = false; - }else if(options.trim !== true){ - throw new Error(`Invalid Option: trim must be a boolean, got ${JSON.stringify(options.trim)}`); + }else if(typeof options.ignore_last_delimiters !== 'boolean'){ + throw new CsvError('CSV_INVALID_OPTION_IGNORE_LAST_DELIMITERS', [ + 'Invalid option `ignore_last_delimiters`:', + 'the value must be a boolean value or an integer,', + `got ${JSON.stringify(options.ignore_last_delimiters)}` + ], options); + } + if(options.ignore_last_delimiters === true && options.columns === false){ + throw new CsvError('CSV_IGNORE_LAST_DELIMITERS_REQUIRES_COLUMNS', [ + 'The option `ignore_last_delimiters`', + 'requires the activation of the `columns` option' + ], options); + } + // Normalize option `info` + if(options.info === undefined || options.info === null || options.info === false){ + options.info = false; + }else if(options.info !== true){ + throw new Error(`Invalid Option: info must be true, got ${JSON.stringify(options.info)}`); + } + // Normalize option `max_record_size` + if(options.max_record_size === undefined || options.max_record_size === null || options.max_record_size === false){ + options.max_record_size = 0; + }else if(Number.isInteger(options.max_record_size) && options.max_record_size >= 0);else if(typeof options.max_record_size === 'string' && /\d+/.test(options.max_record_size)){ + options.max_record_size = parseInt(options.max_record_size); + }else { + throw new Error(`Invalid Option: max_record_size must be a positive integer, got ${JSON.stringify(options.max_record_size)}`); + } + // Normalize option `objname` + if(options.objname === undefined || options.objname === null || options.objname === false){ + options.objname = undefined; + }else if(isBuffer(options.objname)){ + if(options.objname.length === 0){ + throw new Error(`Invalid Option: objname must be a non empty buffer`); + } + if(options.encoding === null);else { + options.objname = options.objname.toString(options.encoding); + } + }else if(typeof options.objname === 'string'){ + if(options.objname.length === 0){ + throw new Error(`Invalid Option: objname must be a non empty string`); + } + // Great, nothing to do + }else if(typeof options.objname === 'number');else { + throw new Error(`Invalid Option: objname must be a string or a buffer, got ${options.objname}`); + } + if(options.objname !== undefined){ + if(typeof options.objname === 'number'){ + if(options.columns !== false){ + throw Error('Invalid Option: objname index cannot be combined with columns or be defined as a field'); + } + }else { // A string or a buffer + if(options.columns === false){ + throw Error('Invalid Option: objname field must be combined with columns or be defined as an index'); + } + } + } + // Normalize option `on_record` + if(options.on_record === undefined || options.on_record === null){ + options.on_record = undefined; + }else if(typeof options.on_record !== 'function'){ + throw new CsvError('CSV_INVALID_OPTION_ON_RECORD', [ + 'Invalid option `on_record`:', + 'expect a function,', + `got ${JSON.stringify(options.on_record)}` + ], options); + } + // Normalize option `quote` + if(options.quote === null || options.quote === false || options.quote === ''){ + options.quote = null; + }else { + if(options.quote === undefined || options.quote === true){ + options.quote = Buffer.from('"', options.encoding); + }else if(typeof options.quote === 'string'){ + options.quote = Buffer.from(options.quote, options.encoding); + } + if(!isBuffer(options.quote)){ + throw new Error(`Invalid Option: quote must be a buffer or a string, got ${JSON.stringify(options.quote)}`); + } + } + // Normalize option `raw` + if(options.raw === undefined || options.raw === null || options.raw === false){ + options.raw = false; + }else if(options.raw !== true){ + throw new Error(`Invalid Option: raw must be true, got ${JSON.stringify(options.raw)}`); + } + // Normalize option `record_delimiter` + if(options.record_delimiter === undefined){ + options.record_delimiter = []; + }else if(typeof options.record_delimiter === 'string' || isBuffer(options.record_delimiter)){ + if(options.record_delimiter.length === 0){ + throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ + 'Invalid option `record_delimiter`:', + 'value must be a non empty string or buffer,', + `got ${JSON.stringify(options.record_delimiter)}` + ], options); } - // Normalize options `trim`, `ltrim` and `rtrim` - if(options.trim === true && opts.ltrim !== false){ - options.ltrim = true; - }else if(options.ltrim !== true){ - options.ltrim = false; + options.record_delimiter = [options.record_delimiter]; + }else if(!Array.isArray(options.record_delimiter)){ + throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ + 'Invalid option `record_delimiter`:', + 'value must be a string, a buffer or array of string|buffer,', + `got ${JSON.stringify(options.record_delimiter)}` + ], options); + } + options.record_delimiter = options.record_delimiter.map(function(rd, i){ + if(typeof rd !== 'string' && ! isBuffer(rd)){ + throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ + 'Invalid option `record_delimiter`:', + 'value must be a string, a buffer or array of string|buffer', + `at index ${i},`, + `got ${JSON.stringify(rd)}` + ], options); + }else if(rd.length === 0){ + throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ + 'Invalid option `record_delimiter`:', + 'value must be a non empty string or buffer', + `at index ${i},`, + `got ${JSON.stringify(rd)}` + ], options); } - if(options.trim === true && opts.rtrim !== false){ - options.rtrim = true; - }else if(options.rtrim !== true){ - options.rtrim = false; + if(typeof rd === 'string'){ + rd = Buffer.from(rd, options.encoding); } - // Normalize option `to` - if(options.to === undefined || options.to === null){ - options.to = -1; - }else { - if(typeof options.to === 'string' && /\d+/.test(options.to)){ - options.to = parseInt(options.to); - } - if(Number.isInteger(options.to)){ - if(options.to <= 0){ - throw new Error(`Invalid Option: to must be a positive integer greater than 0, got ${JSON.stringify(opts.to)}`); - } - }else { - throw new Error(`Invalid Option: to must be an integer, got ${JSON.stringify(opts.to)}`); + return rd; + }); + // Normalize option `relax_column_count` + if(typeof options.relax_column_count === 'boolean');else if(options.relax_column_count === undefined || options.relax_column_count === null){ + options.relax_column_count = false; + }else { + throw new Error(`Invalid Option: relax_column_count must be a boolean, got ${JSON.stringify(options.relax_column_count)}`); + } + if(typeof options.relax_column_count_less === 'boolean');else if(options.relax_column_count_less === undefined || options.relax_column_count_less === null){ + options.relax_column_count_less = false; + }else { + throw new Error(`Invalid Option: relax_column_count_less must be a boolean, got ${JSON.stringify(options.relax_column_count_less)}`); + } + if(typeof options.relax_column_count_more === 'boolean');else if(options.relax_column_count_more === undefined || options.relax_column_count_more === null){ + options.relax_column_count_more = false; + }else { + throw new Error(`Invalid Option: relax_column_count_more must be a boolean, got ${JSON.stringify(options.relax_column_count_more)}`); + } + // Normalize option `relax_quotes` + if(typeof options.relax_quotes === 'boolean');else if(options.relax_quotes === undefined || options.relax_quotes === null){ + options.relax_quotes = false; + }else { + throw new Error(`Invalid Option: relax_quotes must be a boolean, got ${JSON.stringify(options.relax_quotes)}`); + } + // Normalize option `skip_empty_lines` + if(typeof options.skip_empty_lines === 'boolean');else if(options.skip_empty_lines === undefined || options.skip_empty_lines === null){ + options.skip_empty_lines = false; + }else { + throw new Error(`Invalid Option: skip_empty_lines must be a boolean, got ${JSON.stringify(options.skip_empty_lines)}`); + } + // Normalize option `skip_records_with_empty_values` + if(typeof options.skip_records_with_empty_values === 'boolean');else if(options.skip_records_with_empty_values === undefined || options.skip_records_with_empty_values === null){ + options.skip_records_with_empty_values = false; + }else { + throw new Error(`Invalid Option: skip_records_with_empty_values must be a boolean, got ${JSON.stringify(options.skip_records_with_empty_values)}`); + } + // Normalize option `skip_records_with_error` + if(typeof options.skip_records_with_error === 'boolean');else if(options.skip_records_with_error === undefined || options.skip_records_with_error === null){ + options.skip_records_with_error = false; + }else { + throw new Error(`Invalid Option: skip_records_with_error must be a boolean, got ${JSON.stringify(options.skip_records_with_error)}`); + } + // Normalize option `rtrim` + if(options.rtrim === undefined || options.rtrim === null || options.rtrim === false){ + options.rtrim = false; + }else if(options.rtrim !== true){ + throw new Error(`Invalid Option: rtrim must be a boolean, got ${JSON.stringify(options.rtrim)}`); + } + // Normalize option `ltrim` + if(options.ltrim === undefined || options.ltrim === null || options.ltrim === false){ + options.ltrim = false; + }else if(options.ltrim !== true){ + throw new Error(`Invalid Option: ltrim must be a boolean, got ${JSON.stringify(options.ltrim)}`); + } + // Normalize option `trim` + if(options.trim === undefined || options.trim === null || options.trim === false){ + options.trim = false; + }else if(options.trim !== true){ + throw new Error(`Invalid Option: trim must be a boolean, got ${JSON.stringify(options.trim)}`); + } + // Normalize options `trim`, `ltrim` and `rtrim` + if(options.trim === true && opts.ltrim !== false){ + options.ltrim = true; + }else if(options.ltrim !== true){ + options.ltrim = false; + } + if(options.trim === true && opts.rtrim !== false){ + options.rtrim = true; + }else if(options.rtrim !== true){ + options.rtrim = false; + } + // Normalize option `to` + if(options.to === undefined || options.to === null){ + options.to = -1; + }else { + if(typeof options.to === 'string' && /\d+/.test(options.to)){ + options.to = parseInt(options.to); + } + if(Number.isInteger(options.to)){ + if(options.to <= 0){ + throw new Error(`Invalid Option: to must be a positive integer greater than 0, got ${JSON.stringify(opts.to)}`); } - } - // Normalize option `to_line` - if(options.to_line === undefined || options.to_line === null){ - options.to_line = -1; }else { - if(typeof options.to_line === 'string' && /\d+/.test(options.to_line)){ - options.to_line = parseInt(options.to_line); - } - if(Number.isInteger(options.to_line)){ - if(options.to_line <= 0){ - throw new Error(`Invalid Option: to_line must be a positive integer greater than 0, got ${JSON.stringify(opts.to_line)}`); - } - }else { - throw new Error(`Invalid Option: to_line must be an integer, got ${JSON.stringify(opts.to_line)}`); - } - } - this.info = { - bytes: 0, - comment_lines: 0, - empty_lines: 0, - invalid_field_length: 0, - lines: 1, - records: 0 - }; - this.options = options; - this.state = { - bomSkipped: false, - bufBytesStart: 0, - castField: fnCastField, - commenting: false, - // Current error encountered by a record - error: undefined, - enabled: options.from_line === 1, - escaping: false, - escapeIsQuote: isBuffer(options.escape) && isBuffer(options.quote) && Buffer.compare(options.escape, options.quote) === 0, - // columns can be `false`, `true`, `Array` - expectedRecordLength: Array.isArray(options.columns) ? options.columns.length : undefined, - field: new ResizeableBuffer(20), - firstLineToHeaders: fnFirstLineToHeaders, - needMoreDataSize: Math.max( - // Skip if the remaining buffer smaller than comment - options.comment !== null ? options.comment.length : 0, - // Skip if the remaining buffer can be delimiter - ...options.delimiter.map((delimiter) => delimiter.length), - // Skip if the remaining buffer can be escape sequence - options.quote !== null ? options.quote.length : 0, - ), - previousBuf: undefined, - quoting: false, - stop: false, - rawBuffer: new ResizeableBuffer(100), - record: [], - recordHasError: false, - record_length: 0, - recordDelimiterMaxLength: options.record_delimiter.length === 0 ? 2 : Math.max(...options.record_delimiter.map((v) => v.length)), - trimChars: [Buffer.from(' ', options.encoding)[0], Buffer.from('\t', options.encoding)[0]], - wasQuoting: false, - wasRowDelimiter: false - }; - } - // Implementation of `Transform._transform` - _transform(buf, encoding, callback){ - if(this.state.stop === true){ - return; - } - const err = this.__parse(buf, false); - if(err !== undefined){ - this.state.stop = true; + throw new Error(`Invalid Option: to must be an integer, got ${JSON.stringify(opts.to)}`); } - callback(err); } - // Implementation of `Transform._flush` - _flush(callback){ - if(this.state.stop === true){ - return; + // Normalize option `to_line` + if(options.to_line === undefined || options.to_line === null){ + options.to_line = -1; + }else { + if(typeof options.to_line === 'string' && /\d+/.test(options.to_line)){ + options.to_line = parseInt(options.to_line); } - const err = this.__parse(undefined, true); - callback(err); - } - // Central parser implementation - __parse(nextBuf, end){ - const {bom, comment, escape, from_line, ltrim, max_record_size, quote, raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options; - let {record_delimiter} = this.options; - const {bomSkipped, previousBuf, rawBuffer, escapeIsQuote} = this.state; - let buf; - if(previousBuf === undefined){ - if(nextBuf === undefined){ - // Handle empty string - this.push(null); - return; - }else { - buf = nextBuf; + if(Number.isInteger(options.to_line)){ + if(options.to_line <= 0){ + throw new Error(`Invalid Option: to_line must be a positive integer greater than 0, got ${JSON.stringify(opts.to_line)}`); } - }else if(previousBuf !== undefined && nextBuf === undefined){ - buf = previousBuf; }else { - buf = Buffer.concat([previousBuf, nextBuf]); + throw new Error(`Invalid Option: to_line must be an integer, got ${JSON.stringify(opts.to_line)}`); } - // Handle UTF BOM - if(bomSkipped === false){ - if(bom === false){ - this.state.bomSkipped = true; - }else if(buf.length < 3){ - // No enough data - if(end === false){ - // Wait for more data - this.state.previousBuf = buf; + } + return options; + }; + + const isRecordEmpty = function(record){ + return record.every((field) => field == null || field.toString && field.toString().trim() === ''); + }; + + // white space characters + // https://en.wikipedia.org/wiki/Whitespace_character + // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions/Character_Classes#Types + // \f\n\r\t\v\u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff + const tab = 9; + const nl = 10; // \n, 0x0A in hexadecimal, 10 in decimal + const np = 12; + const cr = 13; // \r, 0x0D in hexadécimal, 13 in decimal + const space = 32; + const boms = { + // Note, the following are equals: + // Buffer.from("\ufeff") + // Buffer.from([239, 187, 191]) + // Buffer.from('EFBBBF', 'hex') + 'utf8': Buffer.from([239, 187, 191]), + // Note, the following are equals: + // Buffer.from "\ufeff", 'utf16le + // Buffer.from([255, 254]) + 'utf16le': Buffer.from([255, 254]) + }; + + const transform = function(original_options = {}) { + const info = { + bytes: 0, + comment_lines: 0, + empty_lines: 0, + invalid_field_length: 0, + lines: 1, + records: 0 + }; + const options = normalize_options(original_options); + return { + info: info, + original_options: original_options, + options: options, + state: init_state(options), + __needMoreData: function(i, bufLen, end){ + if(end) return false; + const {quote} = this.options; + const {quoting, needMoreDataSize, recordDelimiterMaxLength} = this.state; + const numOfCharLeft = bufLen - i - 1; + const requiredLength = Math.max( + needMoreDataSize, + // Skip if the remaining buffer smaller than record delimiter + recordDelimiterMaxLength, + // Skip if the remaining buffer can be record delimiter following the closing quote + // 1 is for quote.length + quoting ? (quote.length + recordDelimiterMaxLength) : 0, + ); + return numOfCharLeft < requiredLength; + }, + // Central parser implementation + parse: function(nextBuf, end, push, close){ + const {bom, comment, escape, from_line, ltrim, max_record_size, quote, raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options; + let {record_delimiter} = this.options; + const {bomSkipped, previousBuf, rawBuffer, escapeIsQuote} = this.state; + let buf; + if(previousBuf === undefined){ + if(nextBuf === undefined){ + // Handle empty string + close(); return; + }else { + buf = nextBuf; } + }else if(previousBuf !== undefined && nextBuf === undefined){ + buf = previousBuf; }else { - for(const encoding in boms){ - if(boms[encoding].compare(buf, 0, boms[encoding].length) === 0){ - // Skip BOM - const bomLength = boms[encoding].length; - this.state.bufBytesStart += bomLength; - buf = buf.slice(bomLength); - // Renormalize original options with the new encoding - this.__normalizeOptions({...this.__originalOptions, encoding: encoding}); - break; + buf = Buffer.concat([previousBuf, nextBuf]); + } + // Handle UTF BOM + if(bomSkipped === false){ + if(bom === false){ + this.state.bomSkipped = true; + }else if(buf.length < 3){ + // No enough data + if(end === false){ + // Wait for more data + this.state.previousBuf = buf; + return; + } + }else { + for(const encoding in boms){ + if(boms[encoding].compare(buf, 0, boms[encoding].length) === 0){ + // Skip BOM + const bomLength = boms[encoding].length; + this.state.bufBytesStart += bomLength; + buf = buf.slice(bomLength); + // Renormalize original options with the new encoding + this.options = normalize_options({...this.original_options, encoding: encoding}); + break; + } } + this.state.bomSkipped = true; } - this.state.bomSkipped = true; - } - } - const bufLen = buf.length; - let pos; - for(pos = 0; pos < bufLen; pos++){ - // Ensure we get enough space to look ahead - // There should be a way to move this out of the loop - if(this.__needMoreData(pos, bufLen, end)){ - break; - } - if(this.state.wasRowDelimiter === true){ - this.info.lines++; - this.state.wasRowDelimiter = false; - } - if(to_line !== -1 && this.info.lines > to_line){ - this.state.stop = true; - this.push(null); - return; } - // Auto discovery of record_delimiter, unix, mac and windows supported - if(this.state.quoting === false && record_delimiter.length === 0){ - const record_delimiterCount = this.__autoDiscoverRecordDelimiter(buf, pos); - if(record_delimiterCount){ - record_delimiter = this.options.record_delimiter; + const bufLen = buf.length; + let pos; + for(pos = 0; pos < bufLen; pos++){ + // Ensure we get enough space to look ahead + // There should be a way to move this out of the loop + if(this.__needMoreData(pos, bufLen, end)){ + break; } - } - const chr = buf[pos]; - if(raw === true){ - rawBuffer.append(chr); - } - if((chr === cr || chr === nl) && this.state.wasRowDelimiter === false){ - this.state.wasRowDelimiter = true; - } - // Previous char was a valid escape char - // treat the current char as a regular char - if(this.state.escaping === true){ - this.state.escaping = false; - }else { - // Escape is only active inside quoted fields - // We are quoting, the char is an escape chr and there is a chr to escape - // if(escape !== null && this.state.quoting === true && chr === escape && pos + 1 < bufLen){ - if(escape !== null && this.state.quoting === true && this.__isEscape(buf, pos, chr) && pos + escape.length < bufLen){ - if(escapeIsQuote){ - if(this.__isQuote(buf, pos+escape.length)){ - this.state.escaping = true; - pos += escape.length - 1; - continue; - } - }else { - this.state.escaping = true; - pos += escape.length - 1; - continue; + if(this.state.wasRowDelimiter === true){ + this.info.lines++; + this.state.wasRowDelimiter = false; + } + if(to_line !== -1 && this.info.lines > to_line){ + this.state.stop = true; + close(); + return; + } + // Auto discovery of record_delimiter, unix, mac and windows supported + if(this.state.quoting === false && record_delimiter.length === 0){ + const record_delimiterCount = this.__autoDiscoverRecordDelimiter(buf, pos); + if(record_delimiterCount){ + record_delimiter = this.options.record_delimiter; } } - // Not currently escaping and chr is a quote - // TODO: need to compare bytes instead of single char - if(this.state.commenting === false && this.__isQuote(buf, pos)){ - if(this.state.quoting === true){ - const nextChr = buf[pos+quote.length]; - const isNextChrTrimable = rtrim && this.__isCharTrimable(nextChr); - const isNextChrComment = comment !== null && this.__compareBytes(comment, buf, pos+quote.length, nextChr); - const isNextChrDelimiter = this.__isDelimiter(buf, pos+quote.length, nextChr); - const isNextChrRecordDelimiter = record_delimiter.length === 0 ? this.__autoDiscoverRecordDelimiter(buf, pos+quote.length) : this.__isRecordDelimiter(nextChr, buf, pos+quote.length); - // Escape a quote - // Treat next char as a regular character - if(escape !== null && this.__isEscape(buf, pos, chr) && this.__isQuote(buf, pos + escape.length)){ + const chr = buf[pos]; + if(raw === true){ + rawBuffer.append(chr); + } + if((chr === cr || chr === nl) && this.state.wasRowDelimiter === false){ + this.state.wasRowDelimiter = true; + } + // Previous char was a valid escape char + // treat the current char as a regular char + if(this.state.escaping === true){ + this.state.escaping = false; + }else { + // Escape is only active inside quoted fields + // We are quoting, the char is an escape chr and there is a chr to escape + // if(escape !== null && this.state.quoting === true && chr === escape && pos + 1 < bufLen){ + if(escape !== null && this.state.quoting === true && this.__isEscape(buf, pos, chr) && pos + escape.length < bufLen){ + if(escapeIsQuote){ + if(this.__isQuote(buf, pos+escape.length)){ + this.state.escaping = true; + pos += escape.length - 1; + continue; + } + }else { + this.state.escaping = true; pos += escape.length - 1; - }else if(!nextChr || isNextChrDelimiter || isNextChrRecordDelimiter || isNextChrComment || isNextChrTrimable){ - this.state.quoting = false; - this.state.wasQuoting = true; - pos += quote.length - 1; continue; - }else if(relax_quotes === false){ - const err = this.__error( - new CsvError('CSV_INVALID_CLOSING_QUOTE', [ - 'Invalid Closing Quote:', - `got "${String.fromCharCode(nextChr)}"`, - `at line ${this.info.lines}`, - 'instead of delimiter, record delimiter, trimable character', - '(if activated) or comment', - ], this.options, this.__infoField()) - ); - if(err !== undefined) return err; - }else { - this.state.quoting = false; - this.state.wasQuoting = true; - this.state.field.prepend(quote); - pos += quote.length - 1; } - }else { - if(this.state.field.length !== 0){ - // In relax_quotes mode, treat opening quote preceded by chrs as regular - if(relax_quotes === false){ + } + // Not currently escaping and chr is a quote + // TODO: need to compare bytes instead of single char + if(this.state.commenting === false && this.__isQuote(buf, pos)){ + if(this.state.quoting === true){ + const nextChr = buf[pos+quote.length]; + const isNextChrTrimable = rtrim && this.__isCharTrimable(nextChr); + const isNextChrComment = comment !== null && this.__compareBytes(comment, buf, pos+quote.length, nextChr); + const isNextChrDelimiter = this.__isDelimiter(buf, pos+quote.length, nextChr); + const isNextChrRecordDelimiter = record_delimiter.length === 0 ? this.__autoDiscoverRecordDelimiter(buf, pos+quote.length) : this.__isRecordDelimiter(nextChr, buf, pos+quote.length); + // Escape a quote + // Treat next char as a regular character + if(escape !== null && this.__isEscape(buf, pos, chr) && this.__isQuote(buf, pos + escape.length)){ + pos += escape.length - 1; + }else if(!nextChr || isNextChrDelimiter || isNextChrRecordDelimiter || isNextChrComment || isNextChrTrimable){ + this.state.quoting = false; + this.state.wasQuoting = true; + pos += quote.length - 1; + continue; + }else if(relax_quotes === false){ const err = this.__error( - new CsvError('INVALID_OPENING_QUOTE', [ - 'Invalid Opening Quote:', - `a quote is found inside a field at line ${this.info.lines}`, - ], this.options, this.__infoField(), { - field: this.state.field, - }) + new CsvError('CSV_INVALID_CLOSING_QUOTE', [ + 'Invalid Closing Quote:', + `got "${String.fromCharCode(nextChr)}"`, + `at line ${this.info.lines}`, + 'instead of delimiter, record delimiter, trimable character', + '(if activated) or comment', + ], this.options, this.__infoField()) ); if(err !== undefined) return err; + }else { + this.state.quoting = false; + this.state.wasQuoting = true; + this.state.field.prepend(quote); + pos += quote.length - 1; } }else { - this.state.quoting = true; - pos += quote.length - 1; - continue; - } - } - } - if(this.state.quoting === false){ - const recordDelimiterLength = this.__isRecordDelimiter(chr, buf, pos); - if(recordDelimiterLength !== 0){ - // Do not emit comments which take a full line - const skipCommentLine = this.state.commenting && (this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0); - if(skipCommentLine){ - this.info.comment_lines++; - // Skip full comment line - }else { - // Activate records emition if above from_line - if(this.state.enabled === false && this.info.lines + (this.state.wasRowDelimiter === true ? 1: 0) >= from_line){ - this.state.enabled = true; - this.__resetField(); - this.__resetRecord(); - pos += recordDelimiterLength - 1; + if(this.state.field.length !== 0){ + // In relax_quotes mode, treat opening quote preceded by chrs as regular + if(relax_quotes === false){ + const err = this.__error( + new CsvError('INVALID_OPENING_QUOTE', [ + 'Invalid Opening Quote:', + `a quote is found inside a field at line ${this.info.lines}`, + ], this.options, this.__infoField(), { + field: this.state.field, + }) + ); + if(err !== undefined) return err; + } + }else { + this.state.quoting = true; + pos += quote.length - 1; continue; } - // Skip if line is empty and skip_empty_lines activated - if(skip_empty_lines === true && this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0){ - this.info.empty_lines++; - pos += recordDelimiterLength - 1; - continue; + } + } + if(this.state.quoting === false){ + const recordDelimiterLength = this.__isRecordDelimiter(chr, buf, pos); + if(recordDelimiterLength !== 0){ + // Do not emit comments which take a full line + const skipCommentLine = this.state.commenting && (this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0); + if(skipCommentLine){ + this.info.comment_lines++; + // Skip full comment line + }else { + // Activate records emition if above from_line + if(this.state.enabled === false && this.info.lines + (this.state.wasRowDelimiter === true ? 1: 0) >= from_line){ + this.state.enabled = true; + this.__resetField(); + this.__resetRecord(); + pos += recordDelimiterLength - 1; + continue; + } + // Skip if line is empty and skip_empty_lines activated + if(skip_empty_lines === true && this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0){ + this.info.empty_lines++; + pos += recordDelimiterLength - 1; + continue; + } + this.info.bytes = this.state.bufBytesStart + pos; + const errField = this.__onField(); + if(errField !== undefined) return errField; + this.info.bytes = this.state.bufBytesStart + pos + recordDelimiterLength; + const errRecord = this.__onRecord(push); + if(errRecord !== undefined) return errRecord; + if(to !== -1 && this.info.records >= to){ + this.state.stop = true; + close(); + return; + } } + this.state.commenting = false; + pos += recordDelimiterLength - 1; + continue; + } + if(this.state.commenting){ + continue; + } + const commentCount = comment === null ? 0 : this.__compareBytes(comment, buf, pos, chr); + if(commentCount !== 0){ + this.state.commenting = true; + continue; + } + const delimiterLength = this.__isDelimiter(buf, pos, chr); + if(delimiterLength !== 0){ this.info.bytes = this.state.bufBytesStart + pos; const errField = this.__onField(); if(errField !== undefined) return errField; - this.info.bytes = this.state.bufBytesStart + pos + recordDelimiterLength; - const errRecord = this.__onRecord(); - if(errRecord !== undefined) return errRecord; - if(to !== -1 && this.info.records >= to){ - this.state.stop = true; - this.push(null); - return; - } + pos += delimiterLength - 1; + continue; } - this.state.commenting = false; - pos += recordDelimiterLength - 1; - continue; - } - if(this.state.commenting){ - continue; } - const commentCount = comment === null ? 0 : this.__compareBytes(comment, buf, pos, chr); - if(commentCount !== 0){ - this.state.commenting = true; - continue; - } - const delimiterLength = this.__isDelimiter(buf, pos, chr); - if(delimiterLength !== 0){ - this.info.bytes = this.state.bufBytesStart + pos; - const errField = this.__onField(); - if(errField !== undefined) return errField; - pos += delimiterLength - 1; - continue; + } + if(this.state.commenting === false){ + if(max_record_size !== 0 && this.state.record_length + this.state.field.length > max_record_size){ + const err = this.__error( + new CsvError('CSV_MAX_RECORD_SIZE', [ + 'Max Record Size:', + 'record exceed the maximum number of tolerated bytes', + `of ${max_record_size}`, + `at line ${this.info.lines}`, + ], this.options, this.__infoField()) + ); + if(err !== undefined) return err; } } - } - if(this.state.commenting === false){ - if(max_record_size !== 0 && this.state.record_length + this.state.field.length > max_record_size){ + const lappend = ltrim === false || this.state.quoting === true || this.state.field.length !== 0 || !this.__isCharTrimable(chr); + // rtrim in non quoting is handle in __onField + const rappend = rtrim === false || this.state.wasQuoting === false; + if(lappend === true && rappend === true){ + this.state.field.append(chr); + }else if(rtrim === true && !this.__isCharTrimable(chr)){ const err = this.__error( - new CsvError('CSV_MAX_RECORD_SIZE', [ - 'Max Record Size:', - 'record exceed the maximum number of tolerated bytes', - `of ${max_record_size}`, + new CsvError('CSV_NON_TRIMABLE_CHAR_AFTER_CLOSING_QUOTE', [ + 'Invalid Closing Quote:', + 'found non trimable byte after quote', `at line ${this.info.lines}`, ], this.options, this.__infoField()) ); if(err !== undefined) return err; } } - const lappend = ltrim === false || this.state.quoting === true || this.state.field.length !== 0 || !this.__isCharTrimable(chr); - // rtrim in non quoting is handle in __onField - const rappend = rtrim === false || this.state.wasQuoting === false; - if(lappend === true && rappend === true){ - this.state.field.append(chr); - }else if(rtrim === true && !this.__isCharTrimable(chr)){ - const err = this.__error( - new CsvError('CSV_NON_TRIMABLE_CHAR_AFTER_CLOSING_QUOTE', [ - 'Invalid Closing Quote:', - 'found non trimable byte after quote', - `at line ${this.info.lines}`, - ], this.options, this.__infoField()) - ); - if(err !== undefined) return err; - } - } - if(end === true){ - // Ensure we are not ending in a quoting state - if(this.state.quoting === true){ - const err = this.__error( - new CsvError('CSV_QUOTE_NOT_CLOSED', [ - 'Quote Not Closed:', - `the parsing is finished with an opening quote at line ${this.info.lines}`, - ], this.options, this.__infoField()) - ); - if(err !== undefined) return err; + if(end === true){ + // Ensure we are not ending in a quoting state + if(this.state.quoting === true){ + const err = this.__error( + new CsvError('CSV_QUOTE_NOT_CLOSED', [ + 'Quote Not Closed:', + `the parsing is finished with an opening quote at line ${this.info.lines}`, + ], this.options, this.__infoField()) + ); + if(err !== undefined) return err; + }else { + // Skip last line if it has no characters + if(this.state.wasQuoting === true || this.state.record.length !== 0 || this.state.field.length !== 0){ + this.info.bytes = this.state.bufBytesStart + pos; + const errField = this.__onField(); + if(errField !== undefined) return errField; + const errRecord = this.__onRecord(push); + if(errRecord !== undefined) return errRecord; + }else if(this.state.wasRowDelimiter === true){ + this.info.empty_lines++; + }else if(this.state.commenting === true){ + this.info.comment_lines++; + } + } }else { - // Skip last line if it has no characters - if(this.state.wasQuoting === true || this.state.record.length !== 0 || this.state.field.length !== 0){ - this.info.bytes = this.state.bufBytesStart + pos; - const errField = this.__onField(); - if(errField !== undefined) return errField; - const errRecord = this.__onRecord(); - if(errRecord !== undefined) return errRecord; - }else if(this.state.wasRowDelimiter === true){ - this.info.empty_lines++; - }else if(this.state.commenting === true){ - this.info.comment_lines++; + this.state.bufBytesStart += pos; + this.state.previousBuf = buf.slice(pos); + } + if(this.state.wasRowDelimiter === true){ + this.info.lines++; + this.state.wasRowDelimiter = false; + } + }, + __onRecord: function(push){ + const {columns, group_columns_by_name, encoding, info, from, relax_column_count, relax_column_count_less, relax_column_count_more, raw, skip_records_with_empty_values} = this.options; + const {enabled, record} = this.state; + if(enabled === false){ + return this.__resetRecord(); + } + // Convert the first line into column names + const recordLength = record.length; + if(columns === true){ + if(skip_records_with_empty_values === true && isRecordEmpty(record)){ + this.__resetRecord(); + return; + } + return this.__firstLineToColumns(record); + } + if(columns === false && this.info.records === 0){ + this.state.expectedRecordLength = recordLength; + } + if(recordLength !== this.state.expectedRecordLength){ + const err = columns === false ? + new CsvError('CSV_RECORD_INCONSISTENT_FIELDS_LENGTH', [ + 'Invalid Record Length:', + `expect ${this.state.expectedRecordLength},`, + `got ${recordLength} on line ${this.info.lines}`, + ], this.options, this.__infoField(), { + record: record, + }) + : + new CsvError('CSV_RECORD_INCONSISTENT_COLUMNS', [ + 'Invalid Record Length:', + `columns length is ${columns.length},`, // rename columns + `got ${recordLength} on line ${this.info.lines}`, + ], this.options, this.__infoField(), { + record: record, + }); + if(relax_column_count === true || + (relax_column_count_less === true && recordLength < this.state.expectedRecordLength) || + (relax_column_count_more === true && recordLength > this.state.expectedRecordLength)){ + this.info.invalid_field_length++; + this.state.error = err; + // Error is undefined with skip_records_with_error + }else { + const finalErr = this.__error(err); + if(finalErr) return finalErr; } } - }else { - this.state.bufBytesStart += pos; - this.state.previousBuf = buf.slice(pos); - } - if(this.state.wasRowDelimiter === true){ - this.info.lines++; - this.state.wasRowDelimiter = false; - } - } - __onRecord(){ - const {columns, group_columns_by_name, encoding, info, from, relax_column_count, relax_column_count_less, relax_column_count_more, raw, skip_records_with_empty_values} = this.options; - const {enabled, record} = this.state; - if(enabled === false){ - return this.__resetRecord(); - } - // Convert the first line into column names - const recordLength = record.length; - if(columns === true){ if(skip_records_with_empty_values === true && isRecordEmpty(record)){ this.__resetRecord(); return; } - return this.__firstLineToColumns(record); - } - if(columns === false && this.info.records === 0){ - this.state.expectedRecordLength = recordLength; - } - if(recordLength !== this.state.expectedRecordLength){ - const err = columns === false ? - new CsvError('CSV_RECORD_INCONSISTENT_FIELDS_LENGTH', [ - 'Invalid Record Length:', - `expect ${this.state.expectedRecordLength},`, - `got ${recordLength} on line ${this.info.lines}`, - ], this.options, this.__infoField(), { - record: record, - }) - : - new CsvError('CSV_RECORD_INCONSISTENT_COLUMNS', [ - 'Invalid Record Length:', - `columns length is ${columns.length},`, // rename columns - `got ${recordLength} on line ${this.info.lines}`, - ], this.options, this.__infoField(), { - record: record, - }); - if(relax_column_count === true || - (relax_column_count_less === true && recordLength < this.state.expectedRecordLength) || - (relax_column_count_more === true && recordLength > this.state.expectedRecordLength)){ - this.info.invalid_field_length++; - this.state.error = err; - // Error is undefined with skip_records_with_error - }else { - const finalErr = this.__error(err); - if(finalErr) return finalErr; + if(this.state.recordHasError === true){ + this.__resetRecord(); + this.state.recordHasError = false; + return; } - } - if(skip_records_with_empty_values === true && isRecordEmpty(record)){ - this.__resetRecord(); - return; - } - if(this.state.recordHasError === true){ - this.__resetRecord(); - this.state.recordHasError = false; - return; - } - this.info.records++; - if(from === 1 || this.info.records >= from){ - const {objname} = this.options; - // With columns, records are object - if(columns !== false){ - const obj = {}; - // Transform record array to an object - for(let i = 0, l = record.length; i < l; i++){ - if(columns[i] === undefined || columns[i].disabled) continue; - // Turn duplicate columns into an array - if (group_columns_by_name === true && obj[columns[i].name] !== undefined) { - if (Array.isArray(obj[columns[i].name])) { - obj[columns[i].name] = obj[columns[i].name].concat(record[i]); + this.info.records++; + if(from === 1 || this.info.records >= from){ + const {objname} = this.options; + // With columns, records are object + if(columns !== false){ + const obj = {}; + // Transform record array to an object + for(let i = 0, l = record.length; i < l; i++){ + if(columns[i] === undefined || columns[i].disabled) continue; + // Turn duplicate columns into an array + if (group_columns_by_name === true && obj[columns[i].name] !== undefined) { + if (Array.isArray(obj[columns[i].name])) { + obj[columns[i].name] = obj[columns[i].name].concat(record[i]); + } else { + obj[columns[i].name] = [obj[columns[i].name], record[i]]; + } } else { - obj[columns[i].name] = [obj[columns[i].name], record[i]]; + obj[columns[i].name] = record[i]; } - } else { - obj[columns[i].name] = record[i]; - } - } - // Without objname (default) - if(raw === true || info === true){ - const extRecord = Object.assign( - {record: obj}, - (raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}), - (info === true ? {info: this.__infoRecord()}: {}) - ); - const err = this.__push( - objname === undefined ? extRecord : [obj[objname], extRecord] - ); - if(err){ - return err; - } - }else { - const err = this.__push( - objname === undefined ? obj : [obj[objname], obj] - ); - if(err){ - return err; } - } - // Without columns, records are array - }else { - if(raw === true || info === true){ - const extRecord = Object.assign( - {record: record}, - raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}, - info === true ? {info: this.__infoRecord()}: {} - ); - const err = this.__push( - objname === undefined ? extRecord : [record[objname], extRecord] - ); - if(err){ - return err; + // Without objname (default) + if(raw === true || info === true){ + const extRecord = Object.assign( + {record: obj}, + (raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}), + (info === true ? {info: this.__infoRecord()}: {}) + ); + const err = this.__push( + objname === undefined ? extRecord : [obj[objname], extRecord] + , push); + if(err){ + return err; + } + }else { + const err = this.__push( + objname === undefined ? obj : [obj[objname], obj] + , push); + if(err){ + return err; + } } + // Without columns, records are array }else { - const err = this.__push( - objname === undefined ? record : [record[objname], record] - ); - if(err){ - return err; + if(raw === true || info === true){ + const extRecord = Object.assign( + {record: record}, + raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}, + info === true ? {info: this.__infoRecord()}: {} + ); + const err = this.__push( + objname === undefined ? extRecord : [record[objname], extRecord] + , push); + if(err){ + return err; + } + }else { + const err = this.__push( + objname === undefined ? record : [record[objname], record] + , push); + if(err){ + return err; + } } } } - } - this.__resetRecord(); - } - __firstLineToColumns(record){ - const {firstLineToHeaders} = this.state; - try{ - const headers = firstLineToHeaders === undefined ? record : firstLineToHeaders.call(null, record); - if(!Array.isArray(headers)){ - return this.__error( - new CsvError('CSV_INVALID_COLUMN_MAPPING', [ - 'Invalid Column Mapping:', - 'expect an array from column function,', - `got ${JSON.stringify(headers)}` - ], this.options, this.__infoField(), { - headers: headers, - }) - ); - } - const normalizedHeaders = normalizeColumnsArray(headers); - this.state.expectedRecordLength = normalizedHeaders.length; - this.options.columns = normalizedHeaders; this.__resetRecord(); - return; - }catch(err){ - return err; - } - } - __resetRecord(){ - if(this.options.raw === true){ - this.state.rawBuffer.reset(); - } - this.state.error = undefined; - this.state.record = []; - this.state.record_length = 0; - } - __onField(){ - const {cast, encoding, rtrim, max_record_size} = this.options; - const {enabled, wasQuoting} = this.state; - // Short circuit for the from_line options - if(enabled === false){ - return this.__resetField(); - } - let field = this.state.field.toString(encoding); - if(rtrim === true && wasQuoting === false){ - field = field.trimRight(); - } - if(cast === true){ - const [err, f] = this.__cast(field); - if(err !== undefined) return err; - field = f; - } - this.state.record.push(field); - // Increment record length if record size must not exceed a limit - if(max_record_size !== 0 && typeof field === 'string'){ - this.state.record_length += field.length; - } - this.__resetField(); - } - __resetField(){ - this.state.field.reset(); - this.state.wasQuoting = false; - } - __push(record){ - const {on_record} = this.options; - if(on_record !== undefined){ - const info = this.__infoRecord(); + }, + __firstLineToColumns: function(record){ + const {firstLineToHeaders} = this.state; try{ - record = on_record.call(null, record, info); + const headers = firstLineToHeaders === undefined ? record : firstLineToHeaders.call(null, record); + if(!Array.isArray(headers)){ + return this.__error( + new CsvError('CSV_INVALID_COLUMN_MAPPING', [ + 'Invalid Column Mapping:', + 'expect an array from column function,', + `got ${JSON.stringify(headers)}` + ], this.options, this.__infoField(), { + headers: headers, + }) + ); + } + const normalizedHeaders = normalize_columns_array(headers); + this.state.expectedRecordLength = normalizedHeaders.length; + this.options.columns = normalizedHeaders; + this.__resetRecord(); + return; }catch(err){ return err; } - if(record === undefined || record === null){ return; } - } - this.push(record); - } - // Return a tuple with the error and the casted value - __cast(field){ - const {columns, relax_column_count} = this.options; - const isColumns = Array.isArray(columns); - // Dont loose time calling cast - // because the final record is an object - // and this field can't be associated to a key present in columns - if(isColumns === true && relax_column_count && this.options.columns.length <= this.state.record.length){ - return [undefined, undefined]; - } - if(this.state.castField !== null){ - try{ + }, + __resetRecord: function(){ + if(this.options.raw === true){ + this.state.rawBuffer.reset(); + } + this.state.error = undefined; + this.state.record = []; + this.state.record_length = 0; + }, + __onField: function(){ + const {cast, encoding, rtrim, max_record_size} = this.options; + const {enabled, wasQuoting} = this.state; + // Short circuit for the from_line options + if(enabled === false){ + return this.__resetField(); + } + let field = this.state.field.toString(encoding); + if(rtrim === true && wasQuoting === false){ + field = field.trimRight(); + } + if(cast === true){ + const [err, f] = this.__cast(field); + if(err !== undefined) return err; + field = f; + } + this.state.record.push(field); + // Increment record length if record size must not exceed a limit + if(max_record_size !== 0 && typeof field === 'string'){ + this.state.record_length += field.length; + } + this.__resetField(); + }, + __resetField: function(){ + this.state.field.reset(); + this.state.wasQuoting = false; + }, + __push: function(record, push){ + const {on_record} = this.options; + if(on_record !== undefined){ + const info = this.__infoRecord(); + try{ + record = on_record.call(null, record, info); + }catch(err){ + return err; + } + if(record === undefined || record === null){ return; } + } + push(record); + }, + // Return a tuple with the error and the casted value + __cast: function(field){ + const {columns, relax_column_count} = this.options; + const isColumns = Array.isArray(columns); + // Dont loose time calling cast + // because the final record is an object + // and this field can't be associated to a key present in columns + if(isColumns === true && relax_column_count && this.options.columns.length <= this.state.record.length){ + return [undefined, undefined]; + } + if(this.state.castField !== null){ + try{ + const info = this.__infoField(); + return [undefined, this.state.castField.call(null, field, info)]; + }catch(err){ + return [err]; + } + } + if(this.__isFloat(field)){ + return [undefined, parseFloat(field)]; + }else if(this.options.cast_date !== false){ const info = this.__infoField(); - return [undefined, this.state.castField.call(null, field, info)]; - }catch(err){ - return [err]; + return [undefined, this.options.cast_date.call(null, field, info)]; + } + return [undefined, field]; + }, + // Helper to test if a character is a space or a line delimiter + __isCharTrimable: function(chr){ + return chr === space || chr === tab || chr === cr || chr === nl || chr === np; + }, + // Keep it in case we implement the `cast_int` option + // __isInt(value){ + // // return Number.isInteger(parseInt(value)) + // // return !isNaN( parseInt( obj ) ); + // return /^(\-|\+)?[1-9][0-9]*$/.test(value) + // } + __isFloat: function(value){ + return (value - parseFloat(value) + 1) >= 0; // Borrowed from jquery + }, + __compareBytes: function(sourceBuf, targetBuf, targetPos, firstByte){ + if(sourceBuf[0] !== firstByte) return 0; + const sourceLength = sourceBuf.length; + for(let i = 1; i < sourceLength; i++){ + if(sourceBuf[i] !== targetBuf[targetPos+i]) return 0; + } + return sourceLength; + }, + __isDelimiter: function(buf, pos, chr){ + const {delimiter, ignore_last_delimiters} = this.options; + if(ignore_last_delimiters === true && this.state.record.length === this.options.columns.length - 1){ + return 0; + }else if(ignore_last_delimiters !== false && typeof ignore_last_delimiters === 'number' && this.state.record.length === ignore_last_delimiters - 1){ + return 0; + } + loop1: for(let i = 0; i < delimiter.length; i++){ + const del = delimiter[i]; + if(del[0] === chr){ + for(let j = 1; j < del.length; j++){ + if(del[j] !== buf[pos+j]) continue loop1; + } + return del.length; + } } - } - if(this.__isFloat(field)){ - return [undefined, parseFloat(field)]; - }else if(this.options.cast_date !== false){ - const info = this.__infoField(); - return [undefined, this.options.cast_date.call(null, field, info)]; - } - return [undefined, field]; - } - // Helper to test if a character is a space or a line delimiter - __isCharTrimable(chr){ - return chr === space || chr === tab || chr === cr || chr === nl || chr === np; - } - // Keep it in case we implement the `cast_int` option - // __isInt(value){ - // // return Number.isInteger(parseInt(value)) - // // return !isNaN( parseInt( obj ) ); - // return /^(\-|\+)?[1-9][0-9]*$/.test(value) - // } - __isFloat(value){ - return (value - parseFloat(value) + 1) >= 0; // Borrowed from jquery - } - __compareBytes(sourceBuf, targetBuf, targetPos, firstByte){ - if(sourceBuf[0] !== firstByte) return 0; - const sourceLength = sourceBuf.length; - for(let i = 1; i < sourceLength; i++){ - if(sourceBuf[i] !== targetBuf[targetPos+i]) return 0; - } - return sourceLength; - } - __needMoreData(i, bufLen, end){ - if(end) return false; - const {quote} = this.options; - const {quoting, needMoreDataSize, recordDelimiterMaxLength} = this.state; - const numOfCharLeft = bufLen - i - 1; - const requiredLength = Math.max( - needMoreDataSize, - // Skip if the remaining buffer smaller than record delimiter - recordDelimiterMaxLength, - // Skip if the remaining buffer can be record delimiter following the closing quote - // 1 is for quote.length - quoting ? (quote.length + recordDelimiterMaxLength) : 0, - ); - return numOfCharLeft < requiredLength; - } - __isDelimiter(buf, pos, chr){ - const {delimiter, ignore_last_delimiters} = this.options; - if(ignore_last_delimiters === true && this.state.record.length === this.options.columns.length - 1){ - return 0; - }else if(ignore_last_delimiters !== false && typeof ignore_last_delimiters === 'number' && this.state.record.length === ignore_last_delimiters - 1){ return 0; - } - loop1: for(let i = 0; i < delimiter.length; i++){ - const del = delimiter[i]; - if(del[0] === chr){ - for(let j = 1; j < del.length; j++){ - if(del[j] !== buf[pos+j]) continue loop1; + }, + __isRecordDelimiter: function(chr, buf, pos){ + const {record_delimiter} = this.options; + const recordDelimiterLength = record_delimiter.length; + loop1: for(let i = 0; i < recordDelimiterLength; i++){ + const rd = record_delimiter[i]; + const rdLength = rd.length; + if(rd[0] !== chr){ + continue; } - return del.length; - } - } - return 0; - } - __isRecordDelimiter(chr, buf, pos){ - const {record_delimiter} = this.options; - const recordDelimiterLength = record_delimiter.length; - loop1: for(let i = 0; i < recordDelimiterLength; i++){ - const rd = record_delimiter[i]; - const rdLength = rd.length; - if(rd[0] !== chr){ - continue; - } - for(let j = 1; j < rdLength; j++){ - if(rd[j] !== buf[pos+j]){ - continue loop1; + for(let j = 1; j < rdLength; j++){ + if(rd[j] !== buf[pos+j]){ + continue loop1; + } } + return rd.length; } - return rd.length; - } - return 0; - } - __isEscape(buf, pos, chr){ - const {escape} = this.options; - if(escape === null) return false; - const l = escape.length; - if(escape[0] === chr){ + return 0; + }, + __isEscape: function(buf, pos, chr){ + const {escape} = this.options; + if(escape === null) return false; + const l = escape.length; + if(escape[0] === chr){ + for(let i = 0; i < l; i++){ + if(escape[i] !== buf[pos+i]){ + return false; + } + } + return true; + } + return false; + }, + __isQuote: function(buf, pos){ + const {quote} = this.options; + if(quote === null) return false; + const l = quote.length; for(let i = 0; i < l; i++){ - if(escape[i] !== buf[pos+i]){ + if(quote[i] !== buf[pos+i]){ return false; } } return true; - } - return false; - } - __isQuote(buf, pos){ - const {quote} = this.options; - if(quote === null) return false; - const l = quote.length; - for(let i = 0; i < l; i++){ - if(quote[i] !== buf[pos+i]){ - return false; - } - } - return true; - } - __autoDiscoverRecordDelimiter(buf, pos){ - const {encoding} = this.options; - const chr = buf[pos]; - if(chr === cr){ - if(buf[pos+1] === nl){ - this.options.record_delimiter.push(Buffer.from('\r\n', encoding)); - this.state.recordDelimiterMaxLength = 2; - return 2; - }else { - this.options.record_delimiter.push(Buffer.from('\r', encoding)); + }, + __autoDiscoverRecordDelimiter: function(buf, pos){ + const {encoding} = this.options; + const chr = buf[pos]; + if(chr === cr){ + if(buf[pos+1] === nl){ + this.options.record_delimiter.push(Buffer.from('\r\n', encoding)); + this.state.recordDelimiterMaxLength = 2; + return 2; + }else { + this.options.record_delimiter.push(Buffer.from('\r', encoding)); + this.state.recordDelimiterMaxLength = 1; + return 1; + } + }else if(chr === nl){ + this.options.record_delimiter.push(Buffer.from('\n', encoding)); this.state.recordDelimiterMaxLength = 1; return 1; } - }else if(chr === nl){ - this.options.record_delimiter.push(Buffer.from('\n', encoding)); - this.state.recordDelimiterMaxLength = 1; - return 1; - } - return 0; - } - __error(msg){ - const {encoding, raw, skip_records_with_error} = this.options; - const err = typeof msg === 'string' ? new Error(msg) : msg; - if(skip_records_with_error){ - this.state.recordHasError = true; - this.emit('skip', err, raw ? this.state.rawBuffer.toString(encoding) : undefined); - return undefined; - }else { - return err; + return 0; + }, + __error: function(msg){ + const {encoding, raw, skip_records_with_error} = this.options; + const err = typeof msg === 'string' ? new Error(msg) : msg; + if(skip_records_with_error){ + this.state.recordHasError = true; + if(this.options.on_skip !== undefined){ + this.options.on_skip(err, raw ? this.state.rawBuffer.toString(encoding) : undefined); + } + // this.emit('skip', err, raw ? this.state.rawBuffer.toString(encoding) : undefined); + return undefined; + }else { + return err; + } + }, + __infoDataSet: function(){ + return { + ...this.info, + columns: this.options.columns + }; + }, + __infoRecord: function(){ + const {columns, raw, encoding} = this.options; + return { + ...this.__infoDataSet(), + error: this.state.error, + header: columns === true, + index: this.state.record.length, + raw: raw ? this.state.rawBuffer.toString(encoding) : undefined + }; + }, + __infoField: function(){ + const {columns} = this.options; + const isColumns = Array.isArray(columns); + return { + ...this.__infoRecord(), + column: isColumns === true ? + (columns.length > this.state.record.length ? + columns[this.state.record.length].name : + null + ) : + this.state.record.length, + quoting: this.state.wasQuoting, + }; } - } - __infoDataSet(){ - return { - ...this.info, - columns: this.options.columns - }; - } - __infoRecord(){ - const {columns, raw, encoding} = this.options; - return { - ...this.__infoDataSet(), - error: this.state.error, - header: columns === true, - index: this.state.record.length, - raw: raw ? this.state.rawBuffer.toString(encoding) : undefined - }; - } - __infoField(){ - const {columns} = this.options; - const isColumns = Array.isArray(columns); - return { - ...this.__infoRecord(), - column: isColumns === true ? - (columns.length > this.state.record.length ? - columns[this.state.record.length].name : - null - ) : - this.state.record.length, - quoting: this.state.wasQuoting, - }; - } - } + }; + }; - const parse = function(data, options={}){ + const parse = function(data, opts={}){ if(typeof data === 'string'){ data = Buffer.from(data); } - const records = options && options.objname ? {} : []; - const parser = new Parser(options); - parser.push = function(record){ - if(record === null){ - return; - } - if(options.objname === undefined) + const records = opts && opts.objname ? {} : []; + const parser = transform(opts); + const push = (record) => { + if(parser.options.objname === undefined) records.push(record); else { records[record[0]] = record[1]; } }; - const err1 = parser.__parse(data, false); + const close = () => {}; + const err1 = parser.parse(data, false, push, close); if(err1 !== undefined) throw err1; - const err2 = parser.__parse(undefined, true); + const err2 = parser.parse(undefined, true, push, close); if(err2 !== undefined) throw err2; return records; }; diff --git a/packages/csv-parse/lib/api/CsvError.js b/packages/csv-parse/lib/api/CsvError.js new file mode 100644 index 000000000..b22a847a9 --- /dev/null +++ b/packages/csv-parse/lib/api/CsvError.js @@ -0,0 +1,19 @@ + +class CsvError extends Error { + constructor(code, message, options, ...contexts) { + if(Array.isArray(message)) message = message.join(' '); + super(message); + if(Error.captureStackTrace !== undefined){ + Error.captureStackTrace(this, CsvError); + } + this.code = code; + for(const context of contexts){ + for(const key in context){ + const value = context[key]; + this[key] = Buffer.isBuffer(value) ? value.toString(options.encoding) : value == null ? value : JSON.parse(JSON.stringify(value)); + } + } + } +} + +export {CsvError}; diff --git a/packages/csv-parse/lib/api/index.js b/packages/csv-parse/lib/api/index.js new file mode 100644 index 000000000..17eb2da82 --- /dev/null +++ b/packages/csv-parse/lib/api/index.js @@ -0,0 +1,704 @@ + +import {normalize_columns_array} from './normalize_columns_array.js'; +import {init_state} from './init_state.js'; +import {normalize_options} from './normalize_options.js'; +import {CsvError} from './CsvError.js'; + +const isRecordEmpty = function(record){ + return record.every((field) => field == null || field.toString && field.toString().trim() === ''); +}; + +// white space characters +// https://en.wikipedia.org/wiki/Whitespace_character +// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions/Character_Classes#Types +// \f\n\r\t\v\u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff +const tab = 9; +const nl = 10; // \n, 0x0A in hexadecimal, 10 in decimal +const np = 12; +const cr = 13; // \r, 0x0D in hexadécimal, 13 in decimal +const space = 32; +const boms = { + // Note, the following are equals: + // Buffer.from("\ufeff") + // Buffer.from([239, 187, 191]) + // Buffer.from('EFBBBF', 'hex') + 'utf8': Buffer.from([239, 187, 191]), + // Note, the following are equals: + // Buffer.from "\ufeff", 'utf16le + // Buffer.from([255, 254]) + 'utf16le': Buffer.from([255, 254]) +}; + +const transform = function(original_options = {}) { + const info = { + bytes: 0, + comment_lines: 0, + empty_lines: 0, + invalid_field_length: 0, + lines: 1, + records: 0 + }; + const options = normalize_options(original_options); + return { + info: info, + original_options: original_options, + options: options, + state: init_state(options), + __needMoreData: function(i, bufLen, end){ + if(end) return false; + const {quote} = this.options; + const {quoting, needMoreDataSize, recordDelimiterMaxLength} = this.state; + const numOfCharLeft = bufLen - i - 1; + const requiredLength = Math.max( + needMoreDataSize, + // Skip if the remaining buffer smaller than record delimiter + recordDelimiterMaxLength, + // Skip if the remaining buffer can be record delimiter following the closing quote + // 1 is for quote.length + quoting ? (quote.length + recordDelimiterMaxLength) : 0, + ); + return numOfCharLeft < requiredLength; + }, + // Central parser implementation + parse: function(nextBuf, end, push, close){ + const {bom, comment, escape, from_line, ltrim, max_record_size, quote, raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options; + let {record_delimiter} = this.options; + const {bomSkipped, previousBuf, rawBuffer, escapeIsQuote} = this.state; + let buf; + if(previousBuf === undefined){ + if(nextBuf === undefined){ + // Handle empty string + close(); + return; + }else{ + buf = nextBuf; + } + }else if(previousBuf !== undefined && nextBuf === undefined){ + buf = previousBuf; + }else{ + buf = Buffer.concat([previousBuf, nextBuf]); + } + // Handle UTF BOM + if(bomSkipped === false){ + if(bom === false){ + this.state.bomSkipped = true; + }else if(buf.length < 3){ + // No enough data + if(end === false){ + // Wait for more data + this.state.previousBuf = buf; + return; + } + }else{ + for(const encoding in boms){ + if(boms[encoding].compare(buf, 0, boms[encoding].length) === 0){ + // Skip BOM + const bomLength = boms[encoding].length; + this.state.bufBytesStart += bomLength; + buf = buf.slice(bomLength); + // Renormalize original options with the new encoding + this.options = normalize_options({...this.original_options, encoding: encoding}); + break; + } + } + this.state.bomSkipped = true; + } + } + const bufLen = buf.length; + let pos; + for(pos = 0; pos < bufLen; pos++){ + // Ensure we get enough space to look ahead + // There should be a way to move this out of the loop + if(this.__needMoreData(pos, bufLen, end)){ + break; + } + if(this.state.wasRowDelimiter === true){ + this.info.lines++; + this.state.wasRowDelimiter = false; + } + if(to_line !== -1 && this.info.lines > to_line){ + this.state.stop = true; + close(); + return; + } + // Auto discovery of record_delimiter, unix, mac and windows supported + if(this.state.quoting === false && record_delimiter.length === 0){ + const record_delimiterCount = this.__autoDiscoverRecordDelimiter(buf, pos); + if(record_delimiterCount){ + record_delimiter = this.options.record_delimiter; + } + } + const chr = buf[pos]; + if(raw === true){ + rawBuffer.append(chr); + } + if((chr === cr || chr === nl) && this.state.wasRowDelimiter === false){ + this.state.wasRowDelimiter = true; + } + // Previous char was a valid escape char + // treat the current char as a regular char + if(this.state.escaping === true){ + this.state.escaping = false; + }else{ + // Escape is only active inside quoted fields + // We are quoting, the char is an escape chr and there is a chr to escape + // if(escape !== null && this.state.quoting === true && chr === escape && pos + 1 < bufLen){ + if(escape !== null && this.state.quoting === true && this.__isEscape(buf, pos, chr) && pos + escape.length < bufLen){ + if(escapeIsQuote){ + if(this.__isQuote(buf, pos+escape.length)){ + this.state.escaping = true; + pos += escape.length - 1; + continue; + } + }else{ + this.state.escaping = true; + pos += escape.length - 1; + continue; + } + } + // Not currently escaping and chr is a quote + // TODO: need to compare bytes instead of single char + if(this.state.commenting === false && this.__isQuote(buf, pos)){ + if(this.state.quoting === true){ + const nextChr = buf[pos+quote.length]; + const isNextChrTrimable = rtrim && this.__isCharTrimable(nextChr); + const isNextChrComment = comment !== null && this.__compareBytes(comment, buf, pos+quote.length, nextChr); + const isNextChrDelimiter = this.__isDelimiter(buf, pos+quote.length, nextChr); + const isNextChrRecordDelimiter = record_delimiter.length === 0 ? this.__autoDiscoverRecordDelimiter(buf, pos+quote.length) : this.__isRecordDelimiter(nextChr, buf, pos+quote.length); + // Escape a quote + // Treat next char as a regular character + if(escape !== null && this.__isEscape(buf, pos, chr) && this.__isQuote(buf, pos + escape.length)){ + pos += escape.length - 1; + }else if(!nextChr || isNextChrDelimiter || isNextChrRecordDelimiter || isNextChrComment || isNextChrTrimable){ + this.state.quoting = false; + this.state.wasQuoting = true; + pos += quote.length - 1; + continue; + }else if(relax_quotes === false){ + const err = this.__error( + new CsvError('CSV_INVALID_CLOSING_QUOTE', [ + 'Invalid Closing Quote:', + `got "${String.fromCharCode(nextChr)}"`, + `at line ${this.info.lines}`, + 'instead of delimiter, record delimiter, trimable character', + '(if activated) or comment', + ], this.options, this.__infoField()) + ); + if(err !== undefined) return err; + }else{ + this.state.quoting = false; + this.state.wasQuoting = true; + this.state.field.prepend(quote); + pos += quote.length - 1; + } + }else{ + if(this.state.field.length !== 0){ + // In relax_quotes mode, treat opening quote preceded by chrs as regular + if(relax_quotes === false){ + const err = this.__error( + new CsvError('INVALID_OPENING_QUOTE', [ + 'Invalid Opening Quote:', + `a quote is found inside a field at line ${this.info.lines}`, + ], this.options, this.__infoField(), { + field: this.state.field, + }) + ); + if(err !== undefined) return err; + } + }else{ + this.state.quoting = true; + pos += quote.length - 1; + continue; + } + } + } + if(this.state.quoting === false){ + const recordDelimiterLength = this.__isRecordDelimiter(chr, buf, pos); + if(recordDelimiterLength !== 0){ + // Do not emit comments which take a full line + const skipCommentLine = this.state.commenting && (this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0); + if(skipCommentLine){ + this.info.comment_lines++; + // Skip full comment line + }else{ + // Activate records emition if above from_line + if(this.state.enabled === false && this.info.lines + (this.state.wasRowDelimiter === true ? 1: 0) >= from_line){ + this.state.enabled = true; + this.__resetField(); + this.__resetRecord(); + pos += recordDelimiterLength - 1; + continue; + } + // Skip if line is empty and skip_empty_lines activated + if(skip_empty_lines === true && this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0){ + this.info.empty_lines++; + pos += recordDelimiterLength - 1; + continue; + } + this.info.bytes = this.state.bufBytesStart + pos; + const errField = this.__onField(); + if(errField !== undefined) return errField; + this.info.bytes = this.state.bufBytesStart + pos + recordDelimiterLength; + const errRecord = this.__onRecord(push); + if(errRecord !== undefined) return errRecord; + if(to !== -1 && this.info.records >= to){ + this.state.stop = true; + close(); + return; + } + } + this.state.commenting = false; + pos += recordDelimiterLength - 1; + continue; + } + if(this.state.commenting){ + continue; + } + const commentCount = comment === null ? 0 : this.__compareBytes(comment, buf, pos, chr); + if(commentCount !== 0){ + this.state.commenting = true; + continue; + } + const delimiterLength = this.__isDelimiter(buf, pos, chr); + if(delimiterLength !== 0){ + this.info.bytes = this.state.bufBytesStart + pos; + const errField = this.__onField(); + if(errField !== undefined) return errField; + pos += delimiterLength - 1; + continue; + } + } + } + if(this.state.commenting === false){ + if(max_record_size !== 0 && this.state.record_length + this.state.field.length > max_record_size){ + const err = this.__error( + new CsvError('CSV_MAX_RECORD_SIZE', [ + 'Max Record Size:', + 'record exceed the maximum number of tolerated bytes', + `of ${max_record_size}`, + `at line ${this.info.lines}`, + ], this.options, this.__infoField()) + ); + if(err !== undefined) return err; + } + } + const lappend = ltrim === false || this.state.quoting === true || this.state.field.length !== 0 || !this.__isCharTrimable(chr); + // rtrim in non quoting is handle in __onField + const rappend = rtrim === false || this.state.wasQuoting === false; + if(lappend === true && rappend === true){ + this.state.field.append(chr); + }else if(rtrim === true && !this.__isCharTrimable(chr)){ + const err = this.__error( + new CsvError('CSV_NON_TRIMABLE_CHAR_AFTER_CLOSING_QUOTE', [ + 'Invalid Closing Quote:', + 'found non trimable byte after quote', + `at line ${this.info.lines}`, + ], this.options, this.__infoField()) + ); + if(err !== undefined) return err; + } + } + if(end === true){ + // Ensure we are not ending in a quoting state + if(this.state.quoting === true){ + const err = this.__error( + new CsvError('CSV_QUOTE_NOT_CLOSED', [ + 'Quote Not Closed:', + `the parsing is finished with an opening quote at line ${this.info.lines}`, + ], this.options, this.__infoField()) + ); + if(err !== undefined) return err; + }else{ + // Skip last line if it has no characters + if(this.state.wasQuoting === true || this.state.record.length !== 0 || this.state.field.length !== 0){ + this.info.bytes = this.state.bufBytesStart + pos; + const errField = this.__onField(); + if(errField !== undefined) return errField; + const errRecord = this.__onRecord(push); + if(errRecord !== undefined) return errRecord; + }else if(this.state.wasRowDelimiter === true){ + this.info.empty_lines++; + }else if(this.state.commenting === true){ + this.info.comment_lines++; + } + } + }else{ + this.state.bufBytesStart += pos; + this.state.previousBuf = buf.slice(pos); + } + if(this.state.wasRowDelimiter === true){ + this.info.lines++; + this.state.wasRowDelimiter = false; + } + }, + __onRecord: function(push){ + const {columns, group_columns_by_name, encoding, info, from, relax_column_count, relax_column_count_less, relax_column_count_more, raw, skip_records_with_empty_values} = this.options; + const {enabled, record} = this.state; + if(enabled === false){ + return this.__resetRecord(); + } + // Convert the first line into column names + const recordLength = record.length; + if(columns === true){ + if(skip_records_with_empty_values === true && isRecordEmpty(record)){ + this.__resetRecord(); + return; + } + return this.__firstLineToColumns(record); + } + if(columns === false && this.info.records === 0){ + this.state.expectedRecordLength = recordLength; + } + if(recordLength !== this.state.expectedRecordLength){ + const err = columns === false ? + new CsvError('CSV_RECORD_INCONSISTENT_FIELDS_LENGTH', [ + 'Invalid Record Length:', + `expect ${this.state.expectedRecordLength},`, + `got ${recordLength} on line ${this.info.lines}`, + ], this.options, this.__infoField(), { + record: record, + }) + : + new CsvError('CSV_RECORD_INCONSISTENT_COLUMNS', [ + 'Invalid Record Length:', + `columns length is ${columns.length},`, // rename columns + `got ${recordLength} on line ${this.info.lines}`, + ], this.options, this.__infoField(), { + record: record, + }); + if(relax_column_count === true || + (relax_column_count_less === true && recordLength < this.state.expectedRecordLength) || + (relax_column_count_more === true && recordLength > this.state.expectedRecordLength)){ + this.info.invalid_field_length++; + this.state.error = err; + // Error is undefined with skip_records_with_error + }else{ + const finalErr = this.__error(err); + if(finalErr) return finalErr; + } + } + if(skip_records_with_empty_values === true && isRecordEmpty(record)){ + this.__resetRecord(); + return; + } + if(this.state.recordHasError === true){ + this.__resetRecord(); + this.state.recordHasError = false; + return; + } + this.info.records++; + if(from === 1 || this.info.records >= from){ + const {objname} = this.options; + // With columns, records are object + if(columns !== false){ + const obj = {}; + // Transform record array to an object + for(let i = 0, l = record.length; i < l; i++){ + if(columns[i] === undefined || columns[i].disabled) continue; + // Turn duplicate columns into an array + if (group_columns_by_name === true && obj[columns[i].name] !== undefined) { + if (Array.isArray(obj[columns[i].name])) { + obj[columns[i].name] = obj[columns[i].name].concat(record[i]); + } else { + obj[columns[i].name] = [obj[columns[i].name], record[i]]; + } + } else { + obj[columns[i].name] = record[i]; + } + } + // Without objname (default) + if(raw === true || info === true){ + const extRecord = Object.assign( + {record: obj}, + (raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}), + (info === true ? {info: this.__infoRecord()}: {}) + ); + const err = this.__push( + objname === undefined ? extRecord : [obj[objname], extRecord] + , push); + if(err){ + return err; + } + }else{ + const err = this.__push( + objname === undefined ? obj : [obj[objname], obj] + , push); + if(err){ + return err; + } + } + // Without columns, records are array + }else{ + if(raw === true || info === true){ + const extRecord = Object.assign( + {record: record}, + raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}, + info === true ? {info: this.__infoRecord()}: {} + ); + const err = this.__push( + objname === undefined ? extRecord : [record[objname], extRecord] + , push); + if(err){ + return err; + } + }else{ + const err = this.__push( + objname === undefined ? record : [record[objname], record] + , push); + if(err){ + return err; + } + } + } + } + this.__resetRecord(); + }, + __firstLineToColumns: function(record){ + const {firstLineToHeaders} = this.state; + try{ + const headers = firstLineToHeaders === undefined ? record : firstLineToHeaders.call(null, record); + if(!Array.isArray(headers)){ + return this.__error( + new CsvError('CSV_INVALID_COLUMN_MAPPING', [ + 'Invalid Column Mapping:', + 'expect an array from column function,', + `got ${JSON.stringify(headers)}` + ], this.options, this.__infoField(), { + headers: headers, + }) + ); + } + const normalizedHeaders = normalize_columns_array(headers); + this.state.expectedRecordLength = normalizedHeaders.length; + this.options.columns = normalizedHeaders; + this.__resetRecord(); + return; + }catch(err){ + return err; + } + }, + __resetRecord: function(){ + if(this.options.raw === true){ + this.state.rawBuffer.reset(); + } + this.state.error = undefined; + this.state.record = []; + this.state.record_length = 0; + }, + __onField: function(){ + const {cast, encoding, rtrim, max_record_size} = this.options; + const {enabled, wasQuoting} = this.state; + // Short circuit for the from_line options + if(enabled === false){ + return this.__resetField(); + } + let field = this.state.field.toString(encoding); + if(rtrim === true && wasQuoting === false){ + field = field.trimRight(); + } + if(cast === true){ + const [err, f] = this.__cast(field); + if(err !== undefined) return err; + field = f; + } + this.state.record.push(field); + // Increment record length if record size must not exceed a limit + if(max_record_size !== 0 && typeof field === 'string'){ + this.state.record_length += field.length; + } + this.__resetField(); + }, + __resetField: function(){ + this.state.field.reset(); + this.state.wasQuoting = false; + }, + __push: function(record, push){ + const {on_record} = this.options; + if(on_record !== undefined){ + const info = this.__infoRecord(); + try{ + record = on_record.call(null, record, info); + }catch(err){ + return err; + } + if(record === undefined || record === null){ return; } + } + push(record); + }, + // Return a tuple with the error and the casted value + __cast: function(field){ + const {columns, relax_column_count} = this.options; + const isColumns = Array.isArray(columns); + // Dont loose time calling cast + // because the final record is an object + // and this field can't be associated to a key present in columns + if(isColumns === true && relax_column_count && this.options.columns.length <= this.state.record.length){ + return [undefined, undefined]; + } + if(this.state.castField !== null){ + try{ + const info = this.__infoField(); + return [undefined, this.state.castField.call(null, field, info)]; + }catch(err){ + return [err]; + } + } + if(this.__isFloat(field)){ + return [undefined, parseFloat(field)]; + }else if(this.options.cast_date !== false){ + const info = this.__infoField(); + return [undefined, this.options.cast_date.call(null, field, info)]; + } + return [undefined, field]; + }, + // Helper to test if a character is a space or a line delimiter + __isCharTrimable: function(chr){ + return chr === space || chr === tab || chr === cr || chr === nl || chr === np; + }, + // Keep it in case we implement the `cast_int` option + // __isInt(value){ + // // return Number.isInteger(parseInt(value)) + // // return !isNaN( parseInt( obj ) ); + // return /^(\-|\+)?[1-9][0-9]*$/.test(value) + // } + __isFloat: function(value){ + return (value - parseFloat(value) + 1) >= 0; // Borrowed from jquery + }, + __compareBytes: function(sourceBuf, targetBuf, targetPos, firstByte){ + if(sourceBuf[0] !== firstByte) return 0; + const sourceLength = sourceBuf.length; + for(let i = 1; i < sourceLength; i++){ + if(sourceBuf[i] !== targetBuf[targetPos+i]) return 0; + } + return sourceLength; + }, + __isDelimiter: function(buf, pos, chr){ + const {delimiter, ignore_last_delimiters} = this.options; + if(ignore_last_delimiters === true && this.state.record.length === this.options.columns.length - 1){ + return 0; + }else if(ignore_last_delimiters !== false && typeof ignore_last_delimiters === 'number' && this.state.record.length === ignore_last_delimiters - 1){ + return 0; + } + loop1: for(let i = 0; i < delimiter.length; i++){ + const del = delimiter[i]; + if(del[0] === chr){ + for(let j = 1; j < del.length; j++){ + if(del[j] !== buf[pos+j]) continue loop1; + } + return del.length; + } + } + return 0; + }, + __isRecordDelimiter: function(chr, buf, pos){ + const {record_delimiter} = this.options; + const recordDelimiterLength = record_delimiter.length; + loop1: for(let i = 0; i < recordDelimiterLength; i++){ + const rd = record_delimiter[i]; + const rdLength = rd.length; + if(rd[0] !== chr){ + continue; + } + for(let j = 1; j < rdLength; j++){ + if(rd[j] !== buf[pos+j]){ + continue loop1; + } + } + return rd.length; + } + return 0; + }, + __isEscape: function(buf, pos, chr){ + const {escape} = this.options; + if(escape === null) return false; + const l = escape.length; + if(escape[0] === chr){ + for(let i = 0; i < l; i++){ + if(escape[i] !== buf[pos+i]){ + return false; + } + } + return true; + } + return false; + }, + __isQuote: function(buf, pos){ + const {quote} = this.options; + if(quote === null) return false; + const l = quote.length; + for(let i = 0; i < l; i++){ + if(quote[i] !== buf[pos+i]){ + return false; + } + } + return true; + }, + __autoDiscoverRecordDelimiter: function(buf, pos){ + const {encoding} = this.options; + const chr = buf[pos]; + if(chr === cr){ + if(buf[pos+1] === nl){ + this.options.record_delimiter.push(Buffer.from('\r\n', encoding)); + this.state.recordDelimiterMaxLength = 2; + return 2; + }else{ + this.options.record_delimiter.push(Buffer.from('\r', encoding)); + this.state.recordDelimiterMaxLength = 1; + return 1; + } + }else if(chr === nl){ + this.options.record_delimiter.push(Buffer.from('\n', encoding)); + this.state.recordDelimiterMaxLength = 1; + return 1; + } + return 0; + }, + __error: function(msg){ + const {encoding, raw, skip_records_with_error} = this.options; + const err = typeof msg === 'string' ? new Error(msg) : msg; + if(skip_records_with_error){ + this.state.recordHasError = true; + if(this.options.on_skip !== undefined){ + this.options.on_skip(err, raw ? this.state.rawBuffer.toString(encoding) : undefined); + } + // this.emit('skip', err, raw ? this.state.rawBuffer.toString(encoding) : undefined); + return undefined; + }else{ + return err; + } + }, + __infoDataSet: function(){ + return { + ...this.info, + columns: this.options.columns + }; + }, + __infoRecord: function(){ + const {columns, raw, encoding} = this.options; + return { + ...this.__infoDataSet(), + error: this.state.error, + header: columns === true, + index: this.state.record.length, + raw: raw ? this.state.rawBuffer.toString(encoding) : undefined + }; + }, + __infoField: function(){ + const {columns} = this.options; + const isColumns = Array.isArray(columns); + return { + ...this.__infoRecord(), + column: isColumns === true ? + (columns.length > this.state.record.length ? + columns[this.state.record.length].name : + null + ) : + this.state.record.length, + quoting: this.state.wasQuoting, + }; + } + }; +}; + + +export {transform, CsvError}; diff --git a/packages/csv-parse/lib/api/init_state.js b/packages/csv-parse/lib/api/init_state.js new file mode 100644 index 000000000..28783e3a5 --- /dev/null +++ b/packages/csv-parse/lib/api/init_state.js @@ -0,0 +1,41 @@ + +import ResizeableBuffer from '../utils/ResizeableBuffer.js'; + +const init_state = function(options){ + return { + bomSkipped: false, + bufBytesStart: 0, + castField: options.cast_function, + commenting: false, + // Current error encountered by a record + error: undefined, + enabled: options.from_line === 1, + escaping: false, + escapeIsQuote: Buffer.isBuffer(options.escape) && Buffer.isBuffer(options.quote) && Buffer.compare(options.escape, options.quote) === 0, + // columns can be `false`, `true`, `Array` + expectedRecordLength: Array.isArray(options.columns) ? options.columns.length : undefined, + field: new ResizeableBuffer(20), + firstLineToHeaders: options.cast_first_line_to_header, + needMoreDataSize: Math.max( + // Skip if the remaining buffer smaller than comment + options.comment !== null ? options.comment.length : 0, + // Skip if the remaining buffer can be delimiter + ...options.delimiter.map((delimiter) => delimiter.length), + // Skip if the remaining buffer can be escape sequence + options.quote !== null ? options.quote.length : 0, + ), + previousBuf: undefined, + quoting: false, + stop: false, + rawBuffer: new ResizeableBuffer(100), + record: [], + recordHasError: false, + record_length: 0, + recordDelimiterMaxLength: options.record_delimiter.length === 0 ? 2 : Math.max(...options.record_delimiter.map((v) => v.length)), + trimChars: [Buffer.from(' ', options.encoding)[0], Buffer.from('\t', options.encoding)[0]], + wasQuoting: false, + wasRowDelimiter: false + }; +}; + +export {init_state}; diff --git a/packages/csv-parse/lib/api/normalize_columns_array.js b/packages/csv-parse/lib/api/normalize_columns_array.js new file mode 100644 index 000000000..5d1f5e4b6 --- /dev/null +++ b/packages/csv-parse/lib/api/normalize_columns_array.js @@ -0,0 +1,33 @@ + +import {CsvError} from './CsvError.js'; +import {is_object} from '../utils/is_object.js'; + +const normalize_columns_array = function(columns){ + const normalizedColumns = []; + for(let i = 0, l = columns.length; i < l; i++){ + const column = columns[i]; + if(column === undefined || column === null || column === false){ + normalizedColumns[i] = { disabled: true }; + }else if(typeof column === 'string'){ + normalizedColumns[i] = { name: column }; + }else if(is_object(column)){ + if(typeof column.name !== 'string'){ + throw new CsvError('CSV_OPTION_COLUMNS_MISSING_NAME', [ + 'Option columns missing name:', + `property "name" is required at position ${i}`, + 'when column is an object literal' + ]); + } + normalizedColumns[i] = column; + }else{ + throw new CsvError('CSV_INVALID_COLUMN_DEFINITION', [ + 'Invalid column definition:', + 'expect a string or a literal object,', + `got ${JSON.stringify(column)} at position ${i}` + ]); + } + } + return normalizedColumns; +}; + +export {normalize_columns_array}; diff --git a/packages/csv-parse/lib/api/normalize_options.js b/packages/csv-parse/lib/api/normalize_options.js new file mode 100644 index 000000000..0f790ccfa --- /dev/null +++ b/packages/csv-parse/lib/api/normalize_options.js @@ -0,0 +1,438 @@ + +import {normalize_columns_array} from './normalize_columns_array.js'; +import {CsvError} from './CsvError.js'; +import {underscore} from '../utils/underscore.js'; + +const normalize_options = function(opts){ + const options = {}; + // Merge with user options + for(const opt in opts){ + options[underscore(opt)] = opts[opt]; + } + // Normalize option `encoding` + // Note: defined first because other options depends on it + // to convert chars/strings into buffers. + if(options.encoding === undefined || options.encoding === true){ + options.encoding = 'utf8'; + }else if(options.encoding === null || options.encoding === false){ + options.encoding = null; + }else if(typeof options.encoding !== 'string' && options.encoding !== null){ + throw new CsvError('CSV_INVALID_OPTION_ENCODING', [ + 'Invalid option encoding:', + 'encoding must be a string or null to return a buffer,', + `got ${JSON.stringify(options.encoding)}` + ], options); + } + // Normalize option `bom` + if(options.bom === undefined || options.bom === null || options.bom === false){ + options.bom = false; + }else if(options.bom !== true){ + throw new CsvError('CSV_INVALID_OPTION_BOM', [ + 'Invalid option bom:', 'bom must be true,', + `got ${JSON.stringify(options.bom)}` + ], options); + } + // Normalize option `cast` + options.cast_function = null; + if(options.cast === undefined || options.cast === null || options.cast === false || options.cast === ''){ + options.cast = undefined; + }else if(typeof options.cast === 'function'){ + options.cast_function = options.cast; + options.cast = true; + }else if(options.cast !== true){ + throw new CsvError('CSV_INVALID_OPTION_CAST', [ + 'Invalid option cast:', 'cast must be true or a function,', + `got ${JSON.stringify(options.cast)}` + ], options); + } + // Normalize option `cast_date` + if(options.cast_date === undefined || options.cast_date === null || options.cast_date === false || options.cast_date === ''){ + options.cast_date = false; + }else if(options.cast_date === true){ + options.cast_date = function(value){ + const date = Date.parse(value); + return !isNaN(date) ? new Date(date) : value; + }; + }else{ + throw new CsvError('CSV_INVALID_OPTION_CAST_DATE', [ + 'Invalid option cast_date:', 'cast_date must be true or a function,', + `got ${JSON.stringify(options.cast_date)}` + ], options); + } + // Normalize option `columns` + options.cast_first_line_to_header = null; + if(options.columns === true){ + // Fields in the first line are converted as-is to columns + options.cast_first_line_to_header = undefined; + }else if(typeof options.columns === 'function'){ + options.cast_first_line_to_header = options.columns; + options.columns = true; + }else if(Array.isArray(options.columns)){ + options.columns = normalize_columns_array(options.columns); + }else if(options.columns === undefined || options.columns === null || options.columns === false){ + options.columns = false; + }else{ + throw new CsvError('CSV_INVALID_OPTION_COLUMNS', [ + 'Invalid option columns:', + 'expect an array, a function or true,', + `got ${JSON.stringify(options.columns)}` + ], options); + } + // Normalize option `group_columns_by_name` + if(options.group_columns_by_name === undefined || options.group_columns_by_name === null || options.group_columns_by_name === false){ + options.group_columns_by_name = false; + }else if(options.group_columns_by_name !== true){ + throw new CsvError('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ + 'Invalid option group_columns_by_name:', + 'expect an boolean,', + `got ${JSON.stringify(options.group_columns_by_name)}` + ], options); + }else if(options.columns === false){ + throw new CsvError('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ + 'Invalid option group_columns_by_name:', + 'the `columns` mode must be activated.' + ], options); + } + // Normalize option `comment` + if(options.comment === undefined || options.comment === null || options.comment === false || options.comment === ''){ + options.comment = null; + }else{ + if(typeof options.comment === 'string'){ + options.comment = Buffer.from(options.comment, options.encoding); + } + if(!Buffer.isBuffer(options.comment)){ + throw new CsvError('CSV_INVALID_OPTION_COMMENT', [ + 'Invalid option comment:', + 'comment must be a buffer or a string,', + `got ${JSON.stringify(options.comment)}` + ], options); + } + } + // Normalize option `delimiter` + const delimiter_json = JSON.stringify(options.delimiter); + if(!Array.isArray(options.delimiter)) options.delimiter = [options.delimiter]; + if(options.delimiter.length === 0){ + throw new CsvError('CSV_INVALID_OPTION_DELIMITER', [ + 'Invalid option delimiter:', + 'delimiter must be a non empty string or buffer or array of string|buffer,', + `got ${delimiter_json}` + ], options); + } + options.delimiter = options.delimiter.map(function(delimiter){ + if(delimiter === undefined || delimiter === null || delimiter === false){ + return Buffer.from(',', options.encoding); + } + if(typeof delimiter === 'string'){ + delimiter = Buffer.from(delimiter, options.encoding); + } + if(!Buffer.isBuffer(delimiter) || delimiter.length === 0){ + throw new CsvError('CSV_INVALID_OPTION_DELIMITER', [ + 'Invalid option delimiter:', + 'delimiter must be a non empty string or buffer or array of string|buffer,', + `got ${delimiter_json}` + ], options); + } + return delimiter; + }); + // Normalize option `escape` + if(options.escape === undefined || options.escape === true){ + options.escape = Buffer.from('"', options.encoding); + }else if(typeof options.escape === 'string'){ + options.escape = Buffer.from(options.escape, options.encoding); + }else if (options.escape === null || options.escape === false){ + options.escape = null; + } + if(options.escape !== null){ + if(!Buffer.isBuffer(options.escape)){ + throw new Error(`Invalid Option: escape must be a buffer, a string or a boolean, got ${JSON.stringify(options.escape)}`); + } + } + // Normalize option `from` + if(options.from === undefined || options.from === null){ + options.from = 1; + }else{ + if(typeof options.from === 'string' && /\d+/.test(options.from)){ + options.from = parseInt(options.from); + } + if(Number.isInteger(options.from)){ + if(options.from < 0){ + throw new Error(`Invalid Option: from must be a positive integer, got ${JSON.stringify(opts.from)}`); + } + }else{ + throw new Error(`Invalid Option: from must be an integer, got ${JSON.stringify(options.from)}`); + } + } + // Normalize option `from_line` + if(options.from_line === undefined || options.from_line === null){ + options.from_line = 1; + }else{ + if(typeof options.from_line === 'string' && /\d+/.test(options.from_line)){ + options.from_line = parseInt(options.from_line); + } + if(Number.isInteger(options.from_line)){ + if(options.from_line <= 0){ + throw new Error(`Invalid Option: from_line must be a positive integer greater than 0, got ${JSON.stringify(opts.from_line)}`); + } + }else{ + throw new Error(`Invalid Option: from_line must be an integer, got ${JSON.stringify(opts.from_line)}`); + } + } + // Normalize options `ignore_last_delimiters` + if(options.ignore_last_delimiters === undefined || options.ignore_last_delimiters === null){ + options.ignore_last_delimiters = false; + }else if(typeof options.ignore_last_delimiters === 'number'){ + options.ignore_last_delimiters = Math.floor(options.ignore_last_delimiters); + if(options.ignore_last_delimiters === 0){ + options.ignore_last_delimiters = false; + } + }else if(typeof options.ignore_last_delimiters !== 'boolean'){ + throw new CsvError('CSV_INVALID_OPTION_IGNORE_LAST_DELIMITERS', [ + 'Invalid option `ignore_last_delimiters`:', + 'the value must be a boolean value or an integer,', + `got ${JSON.stringify(options.ignore_last_delimiters)}` + ], options); + } + if(options.ignore_last_delimiters === true && options.columns === false){ + throw new CsvError('CSV_IGNORE_LAST_DELIMITERS_REQUIRES_COLUMNS', [ + 'The option `ignore_last_delimiters`', + 'requires the activation of the `columns` option' + ], options); + } + // Normalize option `info` + if(options.info === undefined || options.info === null || options.info === false){ + options.info = false; + }else if(options.info !== true){ + throw new Error(`Invalid Option: info must be true, got ${JSON.stringify(options.info)}`); + } + // Normalize option `max_record_size` + if(options.max_record_size === undefined || options.max_record_size === null || options.max_record_size === false){ + options.max_record_size = 0; + }else if(Number.isInteger(options.max_record_size) && options.max_record_size >= 0){ + // Great, nothing to do + }else if(typeof options.max_record_size === 'string' && /\d+/.test(options.max_record_size)){ + options.max_record_size = parseInt(options.max_record_size); + }else{ + throw new Error(`Invalid Option: max_record_size must be a positive integer, got ${JSON.stringify(options.max_record_size)}`); + } + // Normalize option `objname` + if(options.objname === undefined || options.objname === null || options.objname === false){ + options.objname = undefined; + }else if(Buffer.isBuffer(options.objname)){ + if(options.objname.length === 0){ + throw new Error(`Invalid Option: objname must be a non empty buffer`); + } + if(options.encoding === null){ + // Don't call `toString`, leave objname as a buffer + }else{ + options.objname = options.objname.toString(options.encoding); + } + }else if(typeof options.objname === 'string'){ + if(options.objname.length === 0){ + throw new Error(`Invalid Option: objname must be a non empty string`); + } + // Great, nothing to do + }else if(typeof options.objname === 'number'){ + // if(options.objname.length === 0){ + // throw new Error(`Invalid Option: objname must be a non empty string`); + // } + // Great, nothing to do + }else{ + throw new Error(`Invalid Option: objname must be a string or a buffer, got ${options.objname}`); + } + if(options.objname !== undefined){ + if(typeof options.objname === 'number'){ + if(options.columns !== false){ + throw Error('Invalid Option: objname index cannot be combined with columns or be defined as a field'); + } + }else{ // A string or a buffer + if(options.columns === false){ + throw Error('Invalid Option: objname field must be combined with columns or be defined as an index'); + } + } + } + // Normalize option `on_record` + if(options.on_record === undefined || options.on_record === null){ + options.on_record = undefined; + }else if(typeof options.on_record !== 'function'){ + throw new CsvError('CSV_INVALID_OPTION_ON_RECORD', [ + 'Invalid option `on_record`:', + 'expect a function,', + `got ${JSON.stringify(options.on_record)}` + ], options); + } + // Normalize option `quote` + if(options.quote === null || options.quote === false || options.quote === ''){ + options.quote = null; + }else{ + if(options.quote === undefined || options.quote === true){ + options.quote = Buffer.from('"', options.encoding); + }else if(typeof options.quote === 'string'){ + options.quote = Buffer.from(options.quote, options.encoding); + } + if(!Buffer.isBuffer(options.quote)){ + throw new Error(`Invalid Option: quote must be a buffer or a string, got ${JSON.stringify(options.quote)}`); + } + } + // Normalize option `raw` + if(options.raw === undefined || options.raw === null || options.raw === false){ + options.raw = false; + }else if(options.raw !== true){ + throw new Error(`Invalid Option: raw must be true, got ${JSON.stringify(options.raw)}`); + } + // Normalize option `record_delimiter` + if(options.record_delimiter === undefined){ + options.record_delimiter = []; + }else if(typeof options.record_delimiter === 'string' || Buffer.isBuffer(options.record_delimiter)){ + if(options.record_delimiter.length === 0){ + throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ + 'Invalid option `record_delimiter`:', + 'value must be a non empty string or buffer,', + `got ${JSON.stringify(options.record_delimiter)}` + ], options); + } + options.record_delimiter = [options.record_delimiter]; + }else if(!Array.isArray(options.record_delimiter)){ + throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ + 'Invalid option `record_delimiter`:', + 'value must be a string, a buffer or array of string|buffer,', + `got ${JSON.stringify(options.record_delimiter)}` + ], options); + } + options.record_delimiter = options.record_delimiter.map(function(rd, i){ + if(typeof rd !== 'string' && ! Buffer.isBuffer(rd)){ + throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ + 'Invalid option `record_delimiter`:', + 'value must be a string, a buffer or array of string|buffer', + `at index ${i},`, + `got ${JSON.stringify(rd)}` + ], options); + }else if(rd.length === 0){ + throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ + 'Invalid option `record_delimiter`:', + 'value must be a non empty string or buffer', + `at index ${i},`, + `got ${JSON.stringify(rd)}` + ], options); + } + if(typeof rd === 'string'){ + rd = Buffer.from(rd, options.encoding); + } + return rd; + }); + // Normalize option `relax_column_count` + if(typeof options.relax_column_count === 'boolean'){ + // Great, nothing to do + }else if(options.relax_column_count === undefined || options.relax_column_count === null){ + options.relax_column_count = false; + }else{ + throw new Error(`Invalid Option: relax_column_count must be a boolean, got ${JSON.stringify(options.relax_column_count)}`); + } + if(typeof options.relax_column_count_less === 'boolean'){ + // Great, nothing to do + }else if(options.relax_column_count_less === undefined || options.relax_column_count_less === null){ + options.relax_column_count_less = false; + }else{ + throw new Error(`Invalid Option: relax_column_count_less must be a boolean, got ${JSON.stringify(options.relax_column_count_less)}`); + } + if(typeof options.relax_column_count_more === 'boolean'){ + // Great, nothing to do + }else if(options.relax_column_count_more === undefined || options.relax_column_count_more === null){ + options.relax_column_count_more = false; + }else{ + throw new Error(`Invalid Option: relax_column_count_more must be a boolean, got ${JSON.stringify(options.relax_column_count_more)}`); + } + // Normalize option `relax_quotes` + if(typeof options.relax_quotes === 'boolean'){ + // Great, nothing to do + }else if(options.relax_quotes === undefined || options.relax_quotes === null){ + options.relax_quotes = false; + }else{ + throw new Error(`Invalid Option: relax_quotes must be a boolean, got ${JSON.stringify(options.relax_quotes)}`); + } + // Normalize option `skip_empty_lines` + if(typeof options.skip_empty_lines === 'boolean'){ + // Great, nothing to do + }else if(options.skip_empty_lines === undefined || options.skip_empty_lines === null){ + options.skip_empty_lines = false; + }else{ + throw new Error(`Invalid Option: skip_empty_lines must be a boolean, got ${JSON.stringify(options.skip_empty_lines)}`); + } + // Normalize option `skip_records_with_empty_values` + if(typeof options.skip_records_with_empty_values === 'boolean'){ + // Great, nothing to do + }else if(options.skip_records_with_empty_values === undefined || options.skip_records_with_empty_values === null){ + options.skip_records_with_empty_values = false; + }else{ + throw new Error(`Invalid Option: skip_records_with_empty_values must be a boolean, got ${JSON.stringify(options.skip_records_with_empty_values)}`); + } + // Normalize option `skip_records_with_error` + if(typeof options.skip_records_with_error === 'boolean'){ + // Great, nothing to do + }else if(options.skip_records_with_error === undefined || options.skip_records_with_error === null){ + options.skip_records_with_error = false; + }else{ + throw new Error(`Invalid Option: skip_records_with_error must be a boolean, got ${JSON.stringify(options.skip_records_with_error)}`); + } + // Normalize option `rtrim` + if(options.rtrim === undefined || options.rtrim === null || options.rtrim === false){ + options.rtrim = false; + }else if(options.rtrim !== true){ + throw new Error(`Invalid Option: rtrim must be a boolean, got ${JSON.stringify(options.rtrim)}`); + } + // Normalize option `ltrim` + if(options.ltrim === undefined || options.ltrim === null || options.ltrim === false){ + options.ltrim = false; + }else if(options.ltrim !== true){ + throw new Error(`Invalid Option: ltrim must be a boolean, got ${JSON.stringify(options.ltrim)}`); + } + // Normalize option `trim` + if(options.trim === undefined || options.trim === null || options.trim === false){ + options.trim = false; + }else if(options.trim !== true){ + throw new Error(`Invalid Option: trim must be a boolean, got ${JSON.stringify(options.trim)}`); + } + // Normalize options `trim`, `ltrim` and `rtrim` + if(options.trim === true && opts.ltrim !== false){ + options.ltrim = true; + }else if(options.ltrim !== true){ + options.ltrim = false; + } + if(options.trim === true && opts.rtrim !== false){ + options.rtrim = true; + }else if(options.rtrim !== true){ + options.rtrim = false; + } + // Normalize option `to` + if(options.to === undefined || options.to === null){ + options.to = -1; + }else{ + if(typeof options.to === 'string' && /\d+/.test(options.to)){ + options.to = parseInt(options.to); + } + if(Number.isInteger(options.to)){ + if(options.to <= 0){ + throw new Error(`Invalid Option: to must be a positive integer greater than 0, got ${JSON.stringify(opts.to)}`); + } + }else{ + throw new Error(`Invalid Option: to must be an integer, got ${JSON.stringify(opts.to)}`); + } + } + // Normalize option `to_line` + if(options.to_line === undefined || options.to_line === null){ + options.to_line = -1; + }else{ + if(typeof options.to_line === 'string' && /\d+/.test(options.to_line)){ + options.to_line = parseInt(options.to_line); + } + if(Number.isInteger(options.to_line)){ + if(options.to_line <= 0){ + throw new Error(`Invalid Option: to_line must be a positive integer greater than 0, got ${JSON.stringify(opts.to_line)}`); + } + }else{ + throw new Error(`Invalid Option: to_line must be an integer, got ${JSON.stringify(opts.to_line)}`); + } + } + return options; +}; + +export {normalize_options}; diff --git a/packages/csv-parse/lib/index.js b/packages/csv-parse/lib/index.js index c954146d7..3dad4b69b 100644 --- a/packages/csv-parse/lib/index.js +++ b/packages/csv-parse/lib/index.js @@ -7,573 +7,32 @@ additional information. */ import { Transform } from 'stream'; -import ResizeableBuffer from './ResizeableBuffer.js'; - -// white space characters -// https://en.wikipedia.org/wiki/Whitespace_character -// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions/Character_Classes#Types -// \f\n\r\t\v\u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff -const tab = 9; -const nl = 10; // \n, 0x0A in hexadecimal, 10 in decimal -const np = 12; -const cr = 13; // \r, 0x0D in hexadécimal, 13 in decimal -const space = 32; -const boms = { - // Note, the following are equals: - // Buffer.from("\ufeff") - // Buffer.from([239, 187, 191]) - // Buffer.from('EFBBBF', 'hex') - 'utf8': Buffer.from([239, 187, 191]), - // Note, the following are equals: - // Buffer.from "\ufeff", 'utf16le - // Buffer.from([255, 254]) - 'utf16le': Buffer.from([255, 254]) -}; - -class CsvError extends Error { - constructor(code, message, options, ...contexts) { - if(Array.isArray(message)) message = message.join(' '); - super(message); - if(Error.captureStackTrace !== undefined){ - Error.captureStackTrace(this, CsvError); - } - this.code = code; - for(const context of contexts){ - for(const key in context){ - const value = context[key]; - this[key] = Buffer.isBuffer(value) ? value.toString(options.encoding) : value == null ? value : JSON.parse(JSON.stringify(value)); - } - } - } -} - -const underscore = function(str){ - return str.replace(/([A-Z])/g, function(_, match){ - return '_' + match.toLowerCase(); - }); -}; - -const isObject = function(obj){ - return (typeof obj === 'object' && obj !== null && !Array.isArray(obj)); -}; - -const isRecordEmpty = function(record){ - return record.every((field) => field == null || field.toString && field.toString().trim() === ''); -}; - -const normalizeColumnsArray = function(columns){ - const normalizedColumns = []; - for(let i = 0, l = columns.length; i < l; i++){ - const column = columns[i]; - if(column === undefined || column === null || column === false){ - normalizedColumns[i] = { disabled: true }; - }else if(typeof column === 'string'){ - normalizedColumns[i] = { name: column }; - }else if(isObject(column)){ - if(typeof column.name !== 'string'){ - throw new CsvError('CSV_OPTION_COLUMNS_MISSING_NAME', [ - 'Option columns missing name:', - `property "name" is required at position ${i}`, - 'when column is an object literal' - ]); - } - normalizedColumns[i] = column; - }else{ - throw new CsvError('CSV_INVALID_COLUMN_DEFINITION', [ - 'Invalid column definition:', - 'expect a string or a literal object,', - `got ${JSON.stringify(column)} at position ${i}` - ]); - } - } - return normalizedColumns; -}; +import {is_object} from './utils/is_object.js'; +import {transform} from './api/index.js'; +import {CsvError} from './api/CsvError.js'; class Parser extends Transform { constructor(opts = {}){ super({...{readableObjectMode: true}, ...opts, encoding: null}); - this.__originalOptions = opts; - this.__normalizeOptions(opts); - } - __normalizeOptions(opts){ - const options = {}; - // Merge with user options - for(const opt in opts){ - options[underscore(opt)] = opts[opt]; - } - // Normalize option `encoding` - // Note: defined first because other options depends on it - // to convert chars/strings into buffers. - if(options.encoding === undefined || options.encoding === true){ - options.encoding = 'utf8'; - }else if(options.encoding === null || options.encoding === false){ - options.encoding = null; - }else if(typeof options.encoding !== 'string' && options.encoding !== null){ - throw new CsvError('CSV_INVALID_OPTION_ENCODING', [ - 'Invalid option encoding:', - 'encoding must be a string or null to return a buffer,', - `got ${JSON.stringify(options.encoding)}` - ], options); - } - // Normalize option `bom` - if(options.bom === undefined || options.bom === null || options.bom === false){ - options.bom = false; - }else if(options.bom !== true){ - throw new CsvError('CSV_INVALID_OPTION_BOM', [ - 'Invalid option bom:', 'bom must be true,', - `got ${JSON.stringify(options.bom)}` - ], options); - } - // Normalize option `cast` - let fnCastField = null; - if(options.cast === undefined || options.cast === null || options.cast === false || options.cast === ''){ - options.cast = undefined; - }else if(typeof options.cast === 'function'){ - fnCastField = options.cast; - options.cast = true; - }else if(options.cast !== true){ - throw new CsvError('CSV_INVALID_OPTION_CAST', [ - 'Invalid option cast:', 'cast must be true or a function,', - `got ${JSON.stringify(options.cast)}` - ], options); - } - // Normalize option `cast_date` - if(options.cast_date === undefined || options.cast_date === null || options.cast_date === false || options.cast_date === ''){ - options.cast_date = false; - }else if(options.cast_date === true){ - options.cast_date = function(value){ - const date = Date.parse(value); - return !isNaN(date) ? new Date(date) : value; - }; - }else{ - throw new CsvError('CSV_INVALID_OPTION_CAST_DATE', [ - 'Invalid option cast_date:', 'cast_date must be true or a function,', - `got ${JSON.stringify(options.cast_date)}` - ], options); - } - // Normalize option `columns` - let fnFirstLineToHeaders = null; - if(options.columns === true){ - // Fields in the first line are converted as-is to columns - fnFirstLineToHeaders = undefined; - }else if(typeof options.columns === 'function'){ - fnFirstLineToHeaders = options.columns; - options.columns = true; - }else if(Array.isArray(options.columns)){ - options.columns = normalizeColumnsArray(options.columns); - }else if(options.columns === undefined || options.columns === null || options.columns === false){ - options.columns = false; - }else{ - throw new CsvError('CSV_INVALID_OPTION_COLUMNS', [ - 'Invalid option columns:', - 'expect an array, a function or true,', - `got ${JSON.stringify(options.columns)}` - ], options); - } - // Normalize option `group_columns_by_name` - if(options.group_columns_by_name === undefined || options.group_columns_by_name === null || options.group_columns_by_name === false){ - options.group_columns_by_name = false; - }else if(options.group_columns_by_name !== true){ - throw new CsvError('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ - 'Invalid option group_columns_by_name:', - 'expect an boolean,', - `got ${JSON.stringify(options.group_columns_by_name)}` - ], options); - }else if(options.columns === false){ - throw new CsvError('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ - 'Invalid option group_columns_by_name:', - 'the `columns` mode must be activated.' - ], options); - } - // Normalize option `comment` - if(options.comment === undefined || options.comment === null || options.comment === false || options.comment === ''){ - options.comment = null; - }else{ - if(typeof options.comment === 'string'){ - options.comment = Buffer.from(options.comment, options.encoding); - } - if(!Buffer.isBuffer(options.comment)){ - throw new CsvError('CSV_INVALID_OPTION_COMMENT', [ - 'Invalid option comment:', - 'comment must be a buffer or a string,', - `got ${JSON.stringify(options.comment)}` - ], options); - } - } - // Normalize option `delimiter` - const delimiter_json = JSON.stringify(options.delimiter); - if(!Array.isArray(options.delimiter)) options.delimiter = [options.delimiter]; - if(options.delimiter.length === 0){ - throw new CsvError('CSV_INVALID_OPTION_DELIMITER', [ - 'Invalid option delimiter:', - 'delimiter must be a non empty string or buffer or array of string|buffer,', - `got ${delimiter_json}` - ], options); - } - options.delimiter = options.delimiter.map(function(delimiter){ - if(delimiter === undefined || delimiter === null || delimiter === false){ - return Buffer.from(',', options.encoding); - } - if(typeof delimiter === 'string'){ - delimiter = Buffer.from(delimiter, options.encoding); - } - if(!Buffer.isBuffer(delimiter) || delimiter.length === 0){ - throw new CsvError('CSV_INVALID_OPTION_DELIMITER', [ - 'Invalid option delimiter:', - 'delimiter must be a non empty string or buffer or array of string|buffer,', - `got ${delimiter_json}` - ], options); - } - return delimiter; - }); - // Normalize option `escape` - if(options.escape === undefined || options.escape === true){ - options.escape = Buffer.from('"', options.encoding); - }else if(typeof options.escape === 'string'){ - options.escape = Buffer.from(options.escape, options.encoding); - }else if (options.escape === null || options.escape === false){ - options.escape = null; - } - if(options.escape !== null){ - if(!Buffer.isBuffer(options.escape)){ - throw new Error(`Invalid Option: escape must be a buffer, a string or a boolean, got ${JSON.stringify(options.escape)}`); - } - } - // Normalize option `from` - if(options.from === undefined || options.from === null){ - options.from = 1; - }else{ - if(typeof options.from === 'string' && /\d+/.test(options.from)){ - options.from = parseInt(options.from); - } - if(Number.isInteger(options.from)){ - if(options.from < 0){ - throw new Error(`Invalid Option: from must be a positive integer, got ${JSON.stringify(opts.from)}`); - } - }else{ - throw new Error(`Invalid Option: from must be an integer, got ${JSON.stringify(options.from)}`); - } - } - // Normalize option `from_line` - if(options.from_line === undefined || options.from_line === null){ - options.from_line = 1; - }else{ - if(typeof options.from_line === 'string' && /\d+/.test(options.from_line)){ - options.from_line = parseInt(options.from_line); - } - if(Number.isInteger(options.from_line)){ - if(options.from_line <= 0){ - throw new Error(`Invalid Option: from_line must be a positive integer greater than 0, got ${JSON.stringify(opts.from_line)}`); - } - }else{ - throw new Error(`Invalid Option: from_line must be an integer, got ${JSON.stringify(opts.from_line)}`); - } - } - // Normalize options `ignore_last_delimiters` - if(options.ignore_last_delimiters === undefined || options.ignore_last_delimiters === null){ - options.ignore_last_delimiters = false; - }else if(typeof options.ignore_last_delimiters === 'number'){ - options.ignore_last_delimiters = Math.floor(options.ignore_last_delimiters); - if(options.ignore_last_delimiters === 0){ - options.ignore_last_delimiters = false; - } - }else if(typeof options.ignore_last_delimiters !== 'boolean'){ - throw new CsvError('CSV_INVALID_OPTION_IGNORE_LAST_DELIMITERS', [ - 'Invalid option `ignore_last_delimiters`:', - 'the value must be a boolean value or an integer,', - `got ${JSON.stringify(options.ignore_last_delimiters)}` - ], options); - } - if(options.ignore_last_delimiters === true && options.columns === false){ - throw new CsvError('CSV_IGNORE_LAST_DELIMITERS_REQUIRES_COLUMNS', [ - 'The option `ignore_last_delimiters`', - 'requires the activation of the `columns` option' - ], options); - } - // Normalize option `info` - if(options.info === undefined || options.info === null || options.info === false){ - options.info = false; - }else if(options.info !== true){ - throw new Error(`Invalid Option: info must be true, got ${JSON.stringify(options.info)}`); - } - // Normalize option `max_record_size` - if(options.max_record_size === undefined || options.max_record_size === null || options.max_record_size === false){ - options.max_record_size = 0; - }else if(Number.isInteger(options.max_record_size) && options.max_record_size >= 0){ - // Great, nothing to do - }else if(typeof options.max_record_size === 'string' && /\d+/.test(options.max_record_size)){ - options.max_record_size = parseInt(options.max_record_size); - }else{ - throw new Error(`Invalid Option: max_record_size must be a positive integer, got ${JSON.stringify(options.max_record_size)}`); - } - // Normalize option `objname` - if(options.objname === undefined || options.objname === null || options.objname === false){ - options.objname = undefined; - }else if(Buffer.isBuffer(options.objname)){ - if(options.objname.length === 0){ - throw new Error(`Invalid Option: objname must be a non empty buffer`); - } - if(options.encoding === null){ - // Don't call `toString`, leave objname as a buffer - }else{ - options.objname = options.objname.toString(options.encoding); - } - }else if(typeof options.objname === 'string'){ - if(options.objname.length === 0){ - throw new Error(`Invalid Option: objname must be a non empty string`); - } - // Great, nothing to do - }else if(typeof options.objname === 'number'){ - // if(options.objname.length === 0){ - // throw new Error(`Invalid Option: objname must be a non empty string`); - // } - // Great, nothing to do - }else{ - throw new Error(`Invalid Option: objname must be a string or a buffer, got ${options.objname}`); - } - if(options.objname !== undefined){ - if(typeof options.objname === 'number'){ - if(options.columns !== false){ - throw Error('Invalid Option: objname index cannot be combined with columns or be defined as a field'); - } - }else{ // A string or a buffer - if(options.columns === false){ - throw Error('Invalid Option: objname field must be combined with columns or be defined as an index'); - } - } - } - // Normalize option `on_record` - if(options.on_record === undefined || options.on_record === null){ - options.on_record = undefined; - }else if(typeof options.on_record !== 'function'){ - throw new CsvError('CSV_INVALID_OPTION_ON_RECORD', [ - 'Invalid option `on_record`:', - 'expect a function,', - `got ${JSON.stringify(options.on_record)}` - ], options); - } - // Normalize option `quote` - if(options.quote === null || options.quote === false || options.quote === ''){ - options.quote = null; - }else{ - if(options.quote === undefined || options.quote === true){ - options.quote = Buffer.from('"', options.encoding); - }else if(typeof options.quote === 'string'){ - options.quote = Buffer.from(options.quote, options.encoding); - } - if(!Buffer.isBuffer(options.quote)){ - throw new Error(`Invalid Option: quote must be a buffer or a string, got ${JSON.stringify(options.quote)}`); - } - } - // Normalize option `raw` - if(options.raw === undefined || options.raw === null || options.raw === false){ - options.raw = false; - }else if(options.raw !== true){ - throw new Error(`Invalid Option: raw must be true, got ${JSON.stringify(options.raw)}`); - } - // Normalize option `record_delimiter` - if(options.record_delimiter === undefined){ - options.record_delimiter = []; - }else if(typeof options.record_delimiter === 'string' || Buffer.isBuffer(options.record_delimiter)){ - if(options.record_delimiter.length === 0){ - throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ - 'Invalid option `record_delimiter`:', - 'value must be a non empty string or buffer,', - `got ${JSON.stringify(options.record_delimiter)}` - ], options); - } - options.record_delimiter = [options.record_delimiter]; - }else if(!Array.isArray(options.record_delimiter)){ - throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ - 'Invalid option `record_delimiter`:', - 'value must be a string, a buffer or array of string|buffer,', - `got ${JSON.stringify(options.record_delimiter)}` - ], options); - } - options.record_delimiter = options.record_delimiter.map(function(rd, i){ - if(typeof rd !== 'string' && ! Buffer.isBuffer(rd)){ - throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ - 'Invalid option `record_delimiter`:', - 'value must be a string, a buffer or array of string|buffer', - `at index ${i},`, - `got ${JSON.stringify(rd)}` - ], options); - }else if(rd.length === 0){ - throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ - 'Invalid option `record_delimiter`:', - 'value must be a non empty string or buffer', - `at index ${i},`, - `got ${JSON.stringify(rd)}` - ], options); - } - if(typeof rd === 'string'){ - rd = Buffer.from(rd, options.encoding); - } - return rd; - }); - // Normalize option `relax_column_count` - if(typeof options.relax_column_count === 'boolean'){ - // Great, nothing to do - }else if(options.relax_column_count === undefined || options.relax_column_count === null){ - options.relax_column_count = false; - }else{ - throw new Error(`Invalid Option: relax_column_count must be a boolean, got ${JSON.stringify(options.relax_column_count)}`); - } - if(typeof options.relax_column_count_less === 'boolean'){ - // Great, nothing to do - }else if(options.relax_column_count_less === undefined || options.relax_column_count_less === null){ - options.relax_column_count_less = false; - }else{ - throw new Error(`Invalid Option: relax_column_count_less must be a boolean, got ${JSON.stringify(options.relax_column_count_less)}`); - } - if(typeof options.relax_column_count_more === 'boolean'){ - // Great, nothing to do - }else if(options.relax_column_count_more === undefined || options.relax_column_count_more === null){ - options.relax_column_count_more = false; - }else{ - throw new Error(`Invalid Option: relax_column_count_more must be a boolean, got ${JSON.stringify(options.relax_column_count_more)}`); - } - // Normalize option `relax_quotes` - if(typeof options.relax_quotes === 'boolean'){ - // Great, nothing to do - }else if(options.relax_quotes === undefined || options.relax_quotes === null){ - options.relax_quotes = false; - }else{ - throw new Error(`Invalid Option: relax_quotes must be a boolean, got ${JSON.stringify(options.relax_quotes)}`); - } - // Normalize option `skip_empty_lines` - if(typeof options.skip_empty_lines === 'boolean'){ - // Great, nothing to do - }else if(options.skip_empty_lines === undefined || options.skip_empty_lines === null){ - options.skip_empty_lines = false; - }else{ - throw new Error(`Invalid Option: skip_empty_lines must be a boolean, got ${JSON.stringify(options.skip_empty_lines)}`); - } - // Normalize option `skip_records_with_empty_values` - if(typeof options.skip_records_with_empty_values === 'boolean'){ - // Great, nothing to do - }else if(options.skip_records_with_empty_values === undefined || options.skip_records_with_empty_values === null){ - options.skip_records_with_empty_values = false; - }else{ - throw new Error(`Invalid Option: skip_records_with_empty_values must be a boolean, got ${JSON.stringify(options.skip_records_with_empty_values)}`); - } - // Normalize option `skip_records_with_error` - if(typeof options.skip_records_with_error === 'boolean'){ - // Great, nothing to do - }else if(options.skip_records_with_error === undefined || options.skip_records_with_error === null){ - options.skip_records_with_error = false; - }else{ - throw new Error(`Invalid Option: skip_records_with_error must be a boolean, got ${JSON.stringify(options.skip_records_with_error)}`); - } - // Normalize option `rtrim` - if(options.rtrim === undefined || options.rtrim === null || options.rtrim === false){ - options.rtrim = false; - }else if(options.rtrim !== true){ - throw new Error(`Invalid Option: rtrim must be a boolean, got ${JSON.stringify(options.rtrim)}`); - } - // Normalize option `ltrim` - if(options.ltrim === undefined || options.ltrim === null || options.ltrim === false){ - options.ltrim = false; - }else if(options.ltrim !== true){ - throw new Error(`Invalid Option: ltrim must be a boolean, got ${JSON.stringify(options.ltrim)}`); - } - // Normalize option `trim` - if(options.trim === undefined || options.trim === null || options.trim === false){ - options.trim = false; - }else if(options.trim !== true){ - throw new Error(`Invalid Option: trim must be a boolean, got ${JSON.stringify(options.trim)}`); - } - // Normalize options `trim`, `ltrim` and `rtrim` - if(options.trim === true && opts.ltrim !== false){ - options.ltrim = true; - }else if(options.ltrim !== true){ - options.ltrim = false; - } - if(options.trim === true && opts.rtrim !== false){ - options.rtrim = true; - }else if(options.rtrim !== true){ - options.rtrim = false; - } - // Normalize option `to` - if(options.to === undefined || options.to === null){ - options.to = -1; - }else{ - if(typeof options.to === 'string' && /\d+/.test(options.to)){ - options.to = parseInt(options.to); - } - if(Number.isInteger(options.to)){ - if(options.to <= 0){ - throw new Error(`Invalid Option: to must be a positive integer greater than 0, got ${JSON.stringify(opts.to)}`); - } - }else{ - throw new Error(`Invalid Option: to must be an integer, got ${JSON.stringify(opts.to)}`); - } - } - // Normalize option `to_line` - if(options.to_line === undefined || options.to_line === null){ - options.to_line = -1; - }else{ - if(typeof options.to_line === 'string' && /\d+/.test(options.to_line)){ - options.to_line = parseInt(options.to_line); - } - if(Number.isInteger(options.to_line)){ - if(options.to_line <= 0){ - throw new Error(`Invalid Option: to_line must be a positive integer greater than 0, got ${JSON.stringify(opts.to_line)}`); - } - }else{ - throw new Error(`Invalid Option: to_line must be an integer, got ${JSON.stringify(opts.to_line)}`); - } - } - this.info = { - bytes: 0, - comment_lines: 0, - empty_lines: 0, - invalid_field_length: 0, - lines: 1, - records: 0 - }; - this.options = options; - this.state = { - bomSkipped: false, - bufBytesStart: 0, - castField: fnCastField, - commenting: false, - // Current error encountered by a record - error: undefined, - enabled: options.from_line === 1, - escaping: false, - escapeIsQuote: Buffer.isBuffer(options.escape) && Buffer.isBuffer(options.quote) && Buffer.compare(options.escape, options.quote) === 0, - // columns can be `false`, `true`, `Array` - expectedRecordLength: Array.isArray(options.columns) ? options.columns.length : undefined, - field: new ResizeableBuffer(20), - firstLineToHeaders: fnFirstLineToHeaders, - needMoreDataSize: Math.max( - // Skip if the remaining buffer smaller than comment - options.comment !== null ? options.comment.length : 0, - // Skip if the remaining buffer can be delimiter - ...options.delimiter.map((delimiter) => delimiter.length), - // Skip if the remaining buffer can be escape sequence - options.quote !== null ? options.quote.length : 0, - ), - previousBuf: undefined, - quoting: false, - stop: false, - rawBuffer: new ResizeableBuffer(100), - record: [], - recordHasError: false, - record_length: 0, - recordDelimiterMaxLength: options.record_delimiter.length === 0 ? 2 : Math.max(...options.record_delimiter.map((v) => v.length)), - trimChars: [Buffer.from(' ', options.encoding)[0], Buffer.from('\t', options.encoding)[0]], - wasQuoting: false, - wasRowDelimiter: false + this.api = transform(opts); + this.api.options.on_skip = (err, chunk) => { + this.emit('skip', err, chunk); }; + // Backward compatibility + this.state = this.api.state; + this.options = this.api.options; + this.info = this.api.info; } // Implementation of `Transform._transform` _transform(buf, encoding, callback){ if(this.state.stop === true){ return; } - const err = this.__parse(buf, false); + const err = this.api.parse(buf, false, (record) => { + this.push.call(this, record); + }, () => { + this.push.call(this, null); + }); if(err !== undefined){ this.state.stop = true; } @@ -584,659 +43,13 @@ class Parser extends Transform { if(this.state.stop === true){ return; } - const err = this.__parse(undefined, true); + const err = this.api.parse(undefined, true, (record) => { + this.push.call(this, record); + }, () => { + this.push.call(this, null); + }); callback(err); } - // Central parser implementation - __parse(nextBuf, end){ - const {bom, comment, escape, from_line, ltrim, max_record_size, quote, raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options; - let {record_delimiter} = this.options; - const {bomSkipped, previousBuf, rawBuffer, escapeIsQuote} = this.state; - let buf; - if(previousBuf === undefined){ - if(nextBuf === undefined){ - // Handle empty string - this.push(null); - return; - }else{ - buf = nextBuf; - } - }else if(previousBuf !== undefined && nextBuf === undefined){ - buf = previousBuf; - }else{ - buf = Buffer.concat([previousBuf, nextBuf]); - } - // Handle UTF BOM - if(bomSkipped === false){ - if(bom === false){ - this.state.bomSkipped = true; - }else if(buf.length < 3){ - // No enough data - if(end === false){ - // Wait for more data - this.state.previousBuf = buf; - return; - } - }else{ - for(const encoding in boms){ - if(boms[encoding].compare(buf, 0, boms[encoding].length) === 0){ - // Skip BOM - const bomLength = boms[encoding].length; - this.state.bufBytesStart += bomLength; - buf = buf.slice(bomLength); - // Renormalize original options with the new encoding - this.__normalizeOptions({...this.__originalOptions, encoding: encoding}); - break; - } - } - this.state.bomSkipped = true; - } - } - const bufLen = buf.length; - let pos; - for(pos = 0; pos < bufLen; pos++){ - // Ensure we get enough space to look ahead - // There should be a way to move this out of the loop - if(this.__needMoreData(pos, bufLen, end)){ - break; - } - if(this.state.wasRowDelimiter === true){ - this.info.lines++; - this.state.wasRowDelimiter = false; - } - if(to_line !== -1 && this.info.lines > to_line){ - this.state.stop = true; - this.push(null); - return; - } - // Auto discovery of record_delimiter, unix, mac and windows supported - if(this.state.quoting === false && record_delimiter.length === 0){ - const record_delimiterCount = this.__autoDiscoverRecordDelimiter(buf, pos); - if(record_delimiterCount){ - record_delimiter = this.options.record_delimiter; - } - } - const chr = buf[pos]; - if(raw === true){ - rawBuffer.append(chr); - } - if((chr === cr || chr === nl) && this.state.wasRowDelimiter === false){ - this.state.wasRowDelimiter = true; - } - // Previous char was a valid escape char - // treat the current char as a regular char - if(this.state.escaping === true){ - this.state.escaping = false; - }else{ - // Escape is only active inside quoted fields - // We are quoting, the char is an escape chr and there is a chr to escape - // if(escape !== null && this.state.quoting === true && chr === escape && pos + 1 < bufLen){ - if(escape !== null && this.state.quoting === true && this.__isEscape(buf, pos, chr) && pos + escape.length < bufLen){ - if(escapeIsQuote){ - if(this.__isQuote(buf, pos+escape.length)){ - this.state.escaping = true; - pos += escape.length - 1; - continue; - } - }else{ - this.state.escaping = true; - pos += escape.length - 1; - continue; - } - } - // Not currently escaping and chr is a quote - // TODO: need to compare bytes instead of single char - if(this.state.commenting === false && this.__isQuote(buf, pos)){ - if(this.state.quoting === true){ - const nextChr = buf[pos+quote.length]; - const isNextChrTrimable = rtrim && this.__isCharTrimable(nextChr); - const isNextChrComment = comment !== null && this.__compareBytes(comment, buf, pos+quote.length, nextChr); - const isNextChrDelimiter = this.__isDelimiter(buf, pos+quote.length, nextChr); - const isNextChrRecordDelimiter = record_delimiter.length === 0 ? this.__autoDiscoverRecordDelimiter(buf, pos+quote.length) : this.__isRecordDelimiter(nextChr, buf, pos+quote.length); - // Escape a quote - // Treat next char as a regular character - if(escape !== null && this.__isEscape(buf, pos, chr) && this.__isQuote(buf, pos + escape.length)){ - pos += escape.length - 1; - }else if(!nextChr || isNextChrDelimiter || isNextChrRecordDelimiter || isNextChrComment || isNextChrTrimable){ - this.state.quoting = false; - this.state.wasQuoting = true; - pos += quote.length - 1; - continue; - }else if(relax_quotes === false){ - const err = this.__error( - new CsvError('CSV_INVALID_CLOSING_QUOTE', [ - 'Invalid Closing Quote:', - `got "${String.fromCharCode(nextChr)}"`, - `at line ${this.info.lines}`, - 'instead of delimiter, record delimiter, trimable character', - '(if activated) or comment', - ], this.options, this.__infoField()) - ); - if(err !== undefined) return err; - }else{ - this.state.quoting = false; - this.state.wasQuoting = true; - this.state.field.prepend(quote); - pos += quote.length - 1; - } - }else{ - if(this.state.field.length !== 0){ - // In relax_quotes mode, treat opening quote preceded by chrs as regular - if(relax_quotes === false){ - const err = this.__error( - new CsvError('INVALID_OPENING_QUOTE', [ - 'Invalid Opening Quote:', - `a quote is found inside a field at line ${this.info.lines}`, - ], this.options, this.__infoField(), { - field: this.state.field, - }) - ); - if(err !== undefined) return err; - } - }else{ - this.state.quoting = true; - pos += quote.length - 1; - continue; - } - } - } - if(this.state.quoting === false){ - const recordDelimiterLength = this.__isRecordDelimiter(chr, buf, pos); - if(recordDelimiterLength !== 0){ - // Do not emit comments which take a full line - const skipCommentLine = this.state.commenting && (this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0); - if(skipCommentLine){ - this.info.comment_lines++; - // Skip full comment line - }else{ - // Activate records emition if above from_line - if(this.state.enabled === false && this.info.lines + (this.state.wasRowDelimiter === true ? 1: 0) >= from_line){ - this.state.enabled = true; - this.__resetField(); - this.__resetRecord(); - pos += recordDelimiterLength - 1; - continue; - } - // Skip if line is empty and skip_empty_lines activated - if(skip_empty_lines === true && this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0){ - this.info.empty_lines++; - pos += recordDelimiterLength - 1; - continue; - } - this.info.bytes = this.state.bufBytesStart + pos; - const errField = this.__onField(); - if(errField !== undefined) return errField; - this.info.bytes = this.state.bufBytesStart + pos + recordDelimiterLength; - const errRecord = this.__onRecord(); - if(errRecord !== undefined) return errRecord; - if(to !== -1 && this.info.records >= to){ - this.state.stop = true; - this.push(null); - return; - } - } - this.state.commenting = false; - pos += recordDelimiterLength - 1; - continue; - } - if(this.state.commenting){ - continue; - } - const commentCount = comment === null ? 0 : this.__compareBytes(comment, buf, pos, chr); - if(commentCount !== 0){ - this.state.commenting = true; - continue; - } - const delimiterLength = this.__isDelimiter(buf, pos, chr); - if(delimiterLength !== 0){ - this.info.bytes = this.state.bufBytesStart + pos; - const errField = this.__onField(); - if(errField !== undefined) return errField; - pos += delimiterLength - 1; - continue; - } - } - } - if(this.state.commenting === false){ - if(max_record_size !== 0 && this.state.record_length + this.state.field.length > max_record_size){ - const err = this.__error( - new CsvError('CSV_MAX_RECORD_SIZE', [ - 'Max Record Size:', - 'record exceed the maximum number of tolerated bytes', - `of ${max_record_size}`, - `at line ${this.info.lines}`, - ], this.options, this.__infoField()) - ); - if(err !== undefined) return err; - } - } - const lappend = ltrim === false || this.state.quoting === true || this.state.field.length !== 0 || !this.__isCharTrimable(chr); - // rtrim in non quoting is handle in __onField - const rappend = rtrim === false || this.state.wasQuoting === false; - if(lappend === true && rappend === true){ - this.state.field.append(chr); - }else if(rtrim === true && !this.__isCharTrimable(chr)){ - const err = this.__error( - new CsvError('CSV_NON_TRIMABLE_CHAR_AFTER_CLOSING_QUOTE', [ - 'Invalid Closing Quote:', - 'found non trimable byte after quote', - `at line ${this.info.lines}`, - ], this.options, this.__infoField()) - ); - if(err !== undefined) return err; - } - } - if(end === true){ - // Ensure we are not ending in a quoting state - if(this.state.quoting === true){ - const err = this.__error( - new CsvError('CSV_QUOTE_NOT_CLOSED', [ - 'Quote Not Closed:', - `the parsing is finished with an opening quote at line ${this.info.lines}`, - ], this.options, this.__infoField()) - ); - if(err !== undefined) return err; - }else{ - // Skip last line if it has no characters - if(this.state.wasQuoting === true || this.state.record.length !== 0 || this.state.field.length !== 0){ - this.info.bytes = this.state.bufBytesStart + pos; - const errField = this.__onField(); - if(errField !== undefined) return errField; - const errRecord = this.__onRecord(); - if(errRecord !== undefined) return errRecord; - }else if(this.state.wasRowDelimiter === true){ - this.info.empty_lines++; - }else if(this.state.commenting === true){ - this.info.comment_lines++; - } - } - }else{ - this.state.bufBytesStart += pos; - this.state.previousBuf = buf.slice(pos); - } - if(this.state.wasRowDelimiter === true){ - this.info.lines++; - this.state.wasRowDelimiter = false; - } - } - __onRecord(){ - const {columns, group_columns_by_name, encoding, info, from, relax_column_count, relax_column_count_less, relax_column_count_more, raw, skip_records_with_empty_values} = this.options; - const {enabled, record} = this.state; - if(enabled === false){ - return this.__resetRecord(); - } - // Convert the first line into column names - const recordLength = record.length; - if(columns === true){ - if(skip_records_with_empty_values === true && isRecordEmpty(record)){ - this.__resetRecord(); - return; - } - return this.__firstLineToColumns(record); - } - if(columns === false && this.info.records === 0){ - this.state.expectedRecordLength = recordLength; - } - if(recordLength !== this.state.expectedRecordLength){ - const err = columns === false ? - new CsvError('CSV_RECORD_INCONSISTENT_FIELDS_LENGTH', [ - 'Invalid Record Length:', - `expect ${this.state.expectedRecordLength},`, - `got ${recordLength} on line ${this.info.lines}`, - ], this.options, this.__infoField(), { - record: record, - }) - : - new CsvError('CSV_RECORD_INCONSISTENT_COLUMNS', [ - 'Invalid Record Length:', - `columns length is ${columns.length},`, // rename columns - `got ${recordLength} on line ${this.info.lines}`, - ], this.options, this.__infoField(), { - record: record, - }); - if(relax_column_count === true || - (relax_column_count_less === true && recordLength < this.state.expectedRecordLength) || - (relax_column_count_more === true && recordLength > this.state.expectedRecordLength)){ - this.info.invalid_field_length++; - this.state.error = err; - // Error is undefined with skip_records_with_error - }else{ - const finalErr = this.__error(err); - if(finalErr) return finalErr; - } - } - if(skip_records_with_empty_values === true && isRecordEmpty(record)){ - this.__resetRecord(); - return; - } - if(this.state.recordHasError === true){ - this.__resetRecord(); - this.state.recordHasError = false; - return; - } - this.info.records++; - if(from === 1 || this.info.records >= from){ - const {objname} = this.options; - // With columns, records are object - if(columns !== false){ - const obj = {}; - // Transform record array to an object - for(let i = 0, l = record.length; i < l; i++){ - if(columns[i] === undefined || columns[i].disabled) continue; - // Turn duplicate columns into an array - if (group_columns_by_name === true && obj[columns[i].name] !== undefined) { - if (Array.isArray(obj[columns[i].name])) { - obj[columns[i].name] = obj[columns[i].name].concat(record[i]); - } else { - obj[columns[i].name] = [obj[columns[i].name], record[i]]; - } - } else { - obj[columns[i].name] = record[i]; - } - } - // Without objname (default) - if(raw === true || info === true){ - const extRecord = Object.assign( - {record: obj}, - (raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}), - (info === true ? {info: this.__infoRecord()}: {}) - ); - const err = this.__push( - objname === undefined ? extRecord : [obj[objname], extRecord] - ); - if(err){ - return err; - } - }else{ - const err = this.__push( - objname === undefined ? obj : [obj[objname], obj] - ); - if(err){ - return err; - } - } - // Without columns, records are array - }else{ - if(raw === true || info === true){ - const extRecord = Object.assign( - {record: record}, - raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}, - info === true ? {info: this.__infoRecord()}: {} - ); - const err = this.__push( - objname === undefined ? extRecord : [record[objname], extRecord] - ); - if(err){ - return err; - } - }else{ - const err = this.__push( - objname === undefined ? record : [record[objname], record] - ); - if(err){ - return err; - } - } - } - } - this.__resetRecord(); - } - __firstLineToColumns(record){ - const {firstLineToHeaders} = this.state; - try{ - const headers = firstLineToHeaders === undefined ? record : firstLineToHeaders.call(null, record); - if(!Array.isArray(headers)){ - return this.__error( - new CsvError('CSV_INVALID_COLUMN_MAPPING', [ - 'Invalid Column Mapping:', - 'expect an array from column function,', - `got ${JSON.stringify(headers)}` - ], this.options, this.__infoField(), { - headers: headers, - }) - ); - } - const normalizedHeaders = normalizeColumnsArray(headers); - this.state.expectedRecordLength = normalizedHeaders.length; - this.options.columns = normalizedHeaders; - this.__resetRecord(); - return; - }catch(err){ - return err; - } - } - __resetRecord(){ - if(this.options.raw === true){ - this.state.rawBuffer.reset(); - } - this.state.error = undefined; - this.state.record = []; - this.state.record_length = 0; - } - __onField(){ - const {cast, encoding, rtrim, max_record_size} = this.options; - const {enabled, wasQuoting} = this.state; - // Short circuit for the from_line options - if(enabled === false){ - return this.__resetField(); - } - let field = this.state.field.toString(encoding); - if(rtrim === true && wasQuoting === false){ - field = field.trimRight(); - } - if(cast === true){ - const [err, f] = this.__cast(field); - if(err !== undefined) return err; - field = f; - } - this.state.record.push(field); - // Increment record length if record size must not exceed a limit - if(max_record_size !== 0 && typeof field === 'string'){ - this.state.record_length += field.length; - } - this.__resetField(); - } - __resetField(){ - this.state.field.reset(); - this.state.wasQuoting = false; - } - __push(record){ - const {on_record} = this.options; - if(on_record !== undefined){ - const info = this.__infoRecord(); - try{ - record = on_record.call(null, record, info); - }catch(err){ - return err; - } - if(record === undefined || record === null){ return; } - } - this.push(record); - } - // Return a tuple with the error and the casted value - __cast(field){ - const {columns, relax_column_count} = this.options; - const isColumns = Array.isArray(columns); - // Dont loose time calling cast - // because the final record is an object - // and this field can't be associated to a key present in columns - if(isColumns === true && relax_column_count && this.options.columns.length <= this.state.record.length){ - return [undefined, undefined]; - } - if(this.state.castField !== null){ - try{ - const info = this.__infoField(); - return [undefined, this.state.castField.call(null, field, info)]; - }catch(err){ - return [err]; - } - } - if(this.__isFloat(field)){ - return [undefined, parseFloat(field)]; - }else if(this.options.cast_date !== false){ - const info = this.__infoField(); - return [undefined, this.options.cast_date.call(null, field, info)]; - } - return [undefined, field]; - } - // Helper to test if a character is a space or a line delimiter - __isCharTrimable(chr){ - return chr === space || chr === tab || chr === cr || chr === nl || chr === np; - } - // Keep it in case we implement the `cast_int` option - // __isInt(value){ - // // return Number.isInteger(parseInt(value)) - // // return !isNaN( parseInt( obj ) ); - // return /^(\-|\+)?[1-9][0-9]*$/.test(value) - // } - __isFloat(value){ - return (value - parseFloat(value) + 1) >= 0; // Borrowed from jquery - } - __compareBytes(sourceBuf, targetBuf, targetPos, firstByte){ - if(sourceBuf[0] !== firstByte) return 0; - const sourceLength = sourceBuf.length; - for(let i = 1; i < sourceLength; i++){ - if(sourceBuf[i] !== targetBuf[targetPos+i]) return 0; - } - return sourceLength; - } - __needMoreData(i, bufLen, end){ - if(end) return false; - const {quote} = this.options; - const {quoting, needMoreDataSize, recordDelimiterMaxLength} = this.state; - const numOfCharLeft = bufLen - i - 1; - const requiredLength = Math.max( - needMoreDataSize, - // Skip if the remaining buffer smaller than record delimiter - recordDelimiterMaxLength, - // Skip if the remaining buffer can be record delimiter following the closing quote - // 1 is for quote.length - quoting ? (quote.length + recordDelimiterMaxLength) : 0, - ); - return numOfCharLeft < requiredLength; - } - __isDelimiter(buf, pos, chr){ - const {delimiter, ignore_last_delimiters} = this.options; - if(ignore_last_delimiters === true && this.state.record.length === this.options.columns.length - 1){ - return 0; - }else if(ignore_last_delimiters !== false && typeof ignore_last_delimiters === 'number' && this.state.record.length === ignore_last_delimiters - 1){ - return 0; - } - loop1: for(let i = 0; i < delimiter.length; i++){ - const del = delimiter[i]; - if(del[0] === chr){ - for(let j = 1; j < del.length; j++){ - if(del[j] !== buf[pos+j]) continue loop1; - } - return del.length; - } - } - return 0; - } - __isRecordDelimiter(chr, buf, pos){ - const {record_delimiter} = this.options; - const recordDelimiterLength = record_delimiter.length; - loop1: for(let i = 0; i < recordDelimiterLength; i++){ - const rd = record_delimiter[i]; - const rdLength = rd.length; - if(rd[0] !== chr){ - continue; - } - for(let j = 1; j < rdLength; j++){ - if(rd[j] !== buf[pos+j]){ - continue loop1; - } - } - return rd.length; - } - return 0; - } - __isEscape(buf, pos, chr){ - const {escape} = this.options; - if(escape === null) return false; - const l = escape.length; - if(escape[0] === chr){ - for(let i = 0; i < l; i++){ - if(escape[i] !== buf[pos+i]){ - return false; - } - } - return true; - } - return false; - } - __isQuote(buf, pos){ - const {quote} = this.options; - if(quote === null) return false; - const l = quote.length; - for(let i = 0; i < l; i++){ - if(quote[i] !== buf[pos+i]){ - return false; - } - } - return true; - } - __autoDiscoverRecordDelimiter(buf, pos){ - const {encoding} = this.options; - const chr = buf[pos]; - if(chr === cr){ - if(buf[pos+1] === nl){ - this.options.record_delimiter.push(Buffer.from('\r\n', encoding)); - this.state.recordDelimiterMaxLength = 2; - return 2; - }else{ - this.options.record_delimiter.push(Buffer.from('\r', encoding)); - this.state.recordDelimiterMaxLength = 1; - return 1; - } - }else if(chr === nl){ - this.options.record_delimiter.push(Buffer.from('\n', encoding)); - this.state.recordDelimiterMaxLength = 1; - return 1; - } - return 0; - } - __error(msg){ - const {encoding, raw, skip_records_with_error} = this.options; - const err = typeof msg === 'string' ? new Error(msg) : msg; - if(skip_records_with_error){ - this.state.recordHasError = true; - this.emit('skip', err, raw ? this.state.rawBuffer.toString(encoding) : undefined); - return undefined; - }else{ - return err; - } - } - __infoDataSet(){ - return { - ...this.info, - columns: this.options.columns - }; - } - __infoRecord(){ - const {columns, raw, encoding} = this.options; - return { - ...this.__infoDataSet(), - error: this.state.error, - header: columns === true, - index: this.state.record.length, - raw: raw ? this.state.rawBuffer.toString(encoding) : undefined - }; - } - __infoField(){ - const {columns} = this.options; - const isColumns = Array.isArray(columns); - return { - ...this.__infoRecord(), - column: isColumns === true ? - (columns.length > this.state.record.length ? - columns[this.state.record.length].name : - null - ) : - this.state.record.length, - quoting: this.state.wasQuoting, - }; - } } const parse = function(){ @@ -1246,7 +59,7 @@ const parse = function(){ const type = typeof argument; if(data === undefined && (typeof argument === 'string' || Buffer.isBuffer(argument))){ data = argument; - }else if(options === undefined && isObject(argument)){ + }else if(options === undefined && is_object(argument)){ options = argument; }else if(callback === undefined && type === 'function'){ callback = argument; @@ -1271,10 +84,10 @@ const parse = function(){ } }); parser.on('error', function(err){ - callback(err, undefined, parser.__infoDataSet()); + callback(err, undefined, parser.api.__infoDataSet()); }); parser.on('end', function(){ - callback(undefined, records, parser.__infoDataSet()); + callback(undefined, records, parser.api.__infoDataSet()); }); } if(data !== undefined){ diff --git a/packages/csv-parse/lib/stream.js b/packages/csv-parse/lib/stream.js new file mode 100644 index 000000000..effa7a8d8 --- /dev/null +++ b/packages/csv-parse/lib/stream.js @@ -0,0 +1,27 @@ + +import { + TransformStream, +} from 'node:stream/web'; +import {transform} from './api/index.js'; + +const parse = (opts) => { + const api = transform(opts); + return new TransformStream({ + async transform(chunk, controller) { + const err = api.parse(chunk, false, (record) => { + controller.enqueue(record); + }, () => { + controller.close() + }); + }, + async flush(controller){ + const err = api.parse(undefined, true, (record) => { + controller.enqueue(record); + }, () => { + controller.close() + }); + } + }); +} + +export {parse}; diff --git a/packages/csv-parse/lib/sync.js b/packages/csv-parse/lib/sync.js index 9aa7ee73f..4229ea314 100644 --- a/packages/csv-parse/lib/sync.js +++ b/packages/csv-parse/lib/sync.js @@ -1,29 +1,27 @@ -import { Parser } from './index.js'; +import {CsvError, transform} from './api/index.js'; -const parse = function(data, options={}){ +const parse = function(data, opts={}){ if(typeof data === 'string'){ data = Buffer.from(data); } - const records = options && options.objname ? {} : []; - const parser = new Parser(options); - parser.push = function(record){ - if(record === null){ - return; - } - if(options.objname === undefined) + const records = opts && opts.objname ? {} : []; + const parser = transform(opts); + const push = (record) => { + if(parser.options.objname === undefined) records.push(record); else{ records[record[0]] = record[1]; } }; - const err1 = parser.__parse(data, false); + const close = () => {}; + const err1 = parser.parse(data, false, push, close); if(err1 !== undefined) throw err1; - const err2 = parser.__parse(undefined, true); + const err2 = parser.parse(undefined, true, push, close); if(err2 !== undefined) throw err2; return records; }; // export default parse export { parse }; -export { CsvError } from './index.js'; +export { CsvError }; diff --git a/packages/csv-parse/lib/ResizeableBuffer.js b/packages/csv-parse/lib/utils/ResizeableBuffer.js similarity index 99% rename from packages/csv-parse/lib/ResizeableBuffer.js rename to packages/csv-parse/lib/utils/ResizeableBuffer.js index c335229b3..4979e9199 100644 --- a/packages/csv-parse/lib/ResizeableBuffer.js +++ b/packages/csv-parse/lib/utils/ResizeableBuffer.js @@ -1,5 +1,4 @@ - class ResizeableBuffer{ constructor(size=100){ this.size = size; diff --git a/packages/csv-parse/lib/utils/is_object.js b/packages/csv-parse/lib/utils/is_object.js new file mode 100644 index 000000000..4f958460c --- /dev/null +++ b/packages/csv-parse/lib/utils/is_object.js @@ -0,0 +1,6 @@ + +const is_object = function(obj){ + return (typeof obj === 'object' && obj !== null && !Array.isArray(obj)); +}; + +export {is_object}; diff --git a/packages/csv-parse/lib/utils/underscore.js b/packages/csv-parse/lib/utils/underscore.js new file mode 100644 index 000000000..453816be6 --- /dev/null +++ b/packages/csv-parse/lib/utils/underscore.js @@ -0,0 +1,8 @@ + +const underscore = function(str){ + return str.replace(/([A-Z])/g, function(_, match){ + return '_' + match.toLowerCase(); + }); +}; + +export {underscore}; diff --git a/packages/csv-parse/test/ResizableBuffer.coffee b/packages/csv-parse/test/ResizableBuffer.coffee index 8416cd853..a6b57eb30 100644 --- a/packages/csv-parse/test/ResizableBuffer.coffee +++ b/packages/csv-parse/test/ResizableBuffer.coffee @@ -1,5 +1,5 @@ -import ResizeableBuffer from '../lib/ResizeableBuffer.js' +import ResizeableBuffer from '../lib/utils/ResizeableBuffer.js' describe 'ResizeableBuffer', -> diff --git a/packages/csv-parse/test/api.assert_error.coffee b/packages/csv-parse/test/api.assert_error.coffee index 5fe9c4797..55bd3d732 100644 --- a/packages/csv-parse/test/api.assert_error.coffee +++ b/packages/csv-parse/test/api.assert_error.coffee @@ -1,6 +1,6 @@ import { CsvError } from '../lib/index.js' -import ResizeableBuffer from '../lib/ResizeableBuffer.js' +import ResizeableBuffer from '../lib/utils/ResizeableBuffer.js' export assert_error = (err, assert = {}, exhaustive = false) -> if Array.isArray err diff --git a/packages/csv-parse/test/api.types.ts b/packages/csv-parse/test/api.types.ts index e52dc0b0b..54255faf9 100644 --- a/packages/csv-parse/test/api.types.ts +++ b/packages/csv-parse/test/api.types.ts @@ -28,10 +28,10 @@ describe('API Types', () => { const options: Options = parser.options const keys: string[] = Object.keys(options) keys.sort().should.eql([ - 'bom', 'cast', 'cast_date', 'columns', 'comment', 'delimiter', + 'bom', 'cast', 'cast_date', 'cast_first_line_to_header', 'cast_function', 'columns', 'comment', 'delimiter', 'encoding', 'escape', 'from', 'from_line', 'group_columns_by_name', 'ignore_last_delimiters', 'info', 'ltrim', 'max_record_size', 'objname', - 'on_record', 'quote', 'raw', 'record_delimiter', + 'on_record', 'on_skip', 'quote', 'raw', 'record_delimiter', 'relax_column_count', 'relax_column_count_less', 'relax_column_count_more', 'relax_quotes', 'rtrim', 'skip_empty_lines', 'skip_records_with_empty_values', 'skip_records_with_error', 'to', diff --git a/packages/csv-parse/test/api.web_stream.coffee b/packages/csv-parse/test/api.web_stream.coffee new file mode 100644 index 000000000..7ba7b865e --- /dev/null +++ b/packages/csv-parse/test/api.web_stream.coffee @@ -0,0 +1,54 @@ + +import {generate as generateStream} from 'csv-generate/stream' +import {parse as parseStream} from '../lib/stream.js' +import {parse as parseClassic} from '../lib/index.js' + +describe 'api stream', -> + + it.skip 'perf classic', -> + console.time('classic') + generator = generateClassic({ + objectMode: true, + length: 10000000 + }); + for await record from generator + continue + console.timeEnd('classic') + + it.skip 'perf stream', -> + console.time('stream') + generator = generateStream({ + objectMode: true, + length: 10000000 + }); + reader = generator.getReader() + while true + { done, value } = await reader.read() + break if done + # for await chunk from generator.getReader().read() + # console.log(Buffer.from(chunk).toString()); + console.timeEnd('stream') + + it 'perf stream with iterator', -> + generator = generateStream + objectMode: false, + length: 5 + parser = parseStream() + stream = generator.pipeThrough parser + records = [] + for await record from stream + records.push record + records.length.should.eql 5 + + # it 'perf stream with reader', -> + # generator = generateStream({ + # objectMode: true, + # length: 10 + # }); + # records = [] + # reader = generator.getReader() + # while true + # { done, record } = await reader.read() + # break if done + # records.push record + # records.length.should.eql 10 diff --git a/packages/csv-parse/test/option.cast.coffee b/packages/csv-parse/test/option.cast.coffee index 4d35c8c0c..d83c7a33c 100644 --- a/packages/csv-parse/test/option.cast.coffee +++ b/packages/csv-parse/test/option.cast.coffee @@ -15,7 +15,7 @@ describe 'Option `cast`', -> it 'all columns', (next) -> parse '1,2,3', cast: true, (err, records) -> - records.should.eql [ [1, 2, 3] ] + records.should.eql [ [1, 2, 3] ] unless err next() it 'convert numbers', (next) -> diff --git a/packages/csv-stringify/dist/cjs/index.cjs b/packages/csv-stringify/dist/cjs/index.cjs index 23fce222f..06b3d38ec 100644 --- a/packages/csv-stringify/dist/cjs/index.cjs +++ b/packages/csv-stringify/dist/cjs/index.cjs @@ -4,8 +4,6 @@ Object.defineProperty(exports, '__esModule', { value: true }); var stream = require('stream'); -const bom_utf8 = Buffer.from([239, 187, 191]); - class CsvError extends Error { constructor(code, message, ...contexts) { if(Array.isArray(message)) message = message.join(' '); @@ -23,16 +21,10 @@ class CsvError extends Error { } } -const isObject = function(obj){ +const is_object = function(obj){ return typeof obj === 'object' && obj !== null && ! Array.isArray(obj); }; -const underscore = function(str){ - return str.replace(/([A-Z])/g, function(_, match){ - return '_' + match.toLowerCase(); - }); -}; - // Lodash implementation of `get` const charCodeOfDot = '.'.charCodeAt(0); @@ -110,441 +102,472 @@ const get = function(object, path){ return (index && index === length) ? object : undefined; }; -class Stringifier extends stream.Transform { - constructor(opts = {}){ - super({...{writableObjectMode: true}, ...opts}); - const options = {}; - let err; - // Merge with user options - for(const opt in opts){ - options[underscore(opt)] = opts[opt]; - } - if((err = this.normalize(options)) !== undefined) throw err; - switch(options.record_delimiter){ - case 'auto': - options.record_delimiter = null; - break; - case 'unix': - options.record_delimiter = "\n"; - break; - case 'mac': - options.record_delimiter = "\r"; - break; - case 'windows': - options.record_delimiter = "\r\n"; - break; - case 'ascii': - options.record_delimiter = "\u001e"; - break; - case 'unicode': - options.record_delimiter = "\u2028"; - break; - } - // Expose options - this.options = options; - // Internal state - this.state = { - stop: false - }; - // Information - this.info = { - records: 0 - }; +const normalize_columns = function(columns){ + if(columns === undefined || columns === null){ + return [undefined, undefined]; } - normalize(options){ - // Normalize option `bom` - if(options.bom === undefined || options.bom === null || options.bom === false){ - options.bom = false; - }else if(options.bom !== true){ - return new CsvError('CSV_OPTION_BOOLEAN_INVALID_TYPE', [ - 'option `bom` is optional and must be a boolean value,', - `got ${JSON.stringify(options.bom)}` - ]); - } - // Normalize option `delimiter` - if(options.delimiter === undefined || options.delimiter === null){ - options.delimiter = ','; - }else if(Buffer.isBuffer(options.delimiter)){ - options.delimiter = options.delimiter.toString(); - }else if(typeof options.delimiter !== 'string'){ - return new CsvError('CSV_OPTION_DELIMITER_INVALID_TYPE', [ - 'option `delimiter` must be a buffer or a string,', - `got ${JSON.stringify(options.delimiter)}` - ]); - } - // Normalize option `quote` - if(options.quote === undefined || options.quote === null){ - options.quote = '"'; - }else if(options.quote === true){ - options.quote = '"'; - }else if(options.quote === false){ - options.quote = ''; - }else if (Buffer.isBuffer(options.quote)){ - options.quote = options.quote.toString(); - }else if(typeof options.quote !== 'string'){ - return new CsvError('CSV_OPTION_QUOTE_INVALID_TYPE', [ - 'option `quote` must be a boolean, a buffer or a string,', - `got ${JSON.stringify(options.quote)}` - ]); - } - // Normalize option `quoted` - if(options.quoted === undefined || options.quoted === null){ - options.quoted = false; - } - // Normalize option `quoted_empty` - if(options.quoted_empty === undefined || options.quoted_empty === null){ - options.quoted_empty = undefined; - } - // Normalize option `quoted_match` - if(options.quoted_match === undefined || options.quoted_match === null || options.quoted_match === false){ - options.quoted_match = null; - }else if(!Array.isArray(options.quoted_match)){ - options.quoted_match = [options.quoted_match]; - } - if(options.quoted_match){ - for(const quoted_match of options.quoted_match){ - const isString = typeof quoted_match === 'string'; - const isRegExp = quoted_match instanceof RegExp; - if(!isString && !isRegExp){ - return Error(`Invalid Option: quoted_match must be a string or a regex, got ${JSON.stringify(quoted_match)}`); + if(typeof columns !== 'object'){ + return [Error('Invalid option "columns": expect an array or an object')]; + } + if(!Array.isArray(columns)){ + const newcolumns = []; + for(const k in columns){ + newcolumns.push({ + key: k, + header: columns[k] + }); + } + columns = newcolumns; + }else { + const newcolumns = []; + for(const column of columns){ + if(typeof column === 'string'){ + newcolumns.push({ + key: column, + header: column + }); + }else if(typeof column === 'object' && column !== null && !Array.isArray(column)){ + if(!column.key){ + return [Error('Invalid column definition: property "key" is required')]; + } + if(column.header === undefined){ + column.header = column.key; } + newcolumns.push(column); + }else { + return [Error('Invalid column definition: expect a string or an object')]; } } - // Normalize option `quoted_string` - if(options.quoted_string === undefined || options.quoted_string === null){ - options.quoted_string = false; - } - // Normalize option `eof` - if(options.eof === undefined || options.eof === null){ - options.eof = true; - } - // Normalize option `escape` - if(options.escape === undefined || options.escape === null){ - options.escape = '"'; - }else if(Buffer.isBuffer(options.escape)){ - options.escape = options.escape.toString(); - }else if(typeof options.escape !== 'string'){ - return Error(`Invalid Option: escape must be a buffer or a string, got ${JSON.stringify(options.escape)}`); - } - if (options.escape.length > 1){ - return Error(`Invalid Option: escape must be one character, got ${options.escape.length} characters`); - } - // Normalize option `header` - if(options.header === undefined || options.header === null){ - options.header = false; - } - // Normalize option `columns` - const [err, columns] = this.normalize_columns(options.columns); - if(err) return err; - options.columns = columns; - // Normalize option `quoted` - if(options.quoted === undefined || options.quoted === null){ - options.quoted = false; - } - // Normalize option `cast` - if(options.cast === undefined || options.cast === null){ - options.cast = {}; - } - // Normalize option cast.bigint - if(options.cast.bigint === undefined || options.cast.bigint === null){ - // Cast boolean to string by default - options.cast.bigint = value => '' + value; - } - // Normalize option cast.boolean - if(options.cast.boolean === undefined || options.cast.boolean === null){ - // Cast boolean to string by default - options.cast.boolean = value => value ? '1' : ''; - } - // Normalize option cast.date - if(options.cast.date === undefined || options.cast.date === null){ - // Cast date to timestamp string by default - options.cast.date = value => '' + value.getTime(); - } - // Normalize option cast.number - if(options.cast.number === undefined || options.cast.number === null){ - // Cast number to string using native casting by default - options.cast.number = value => '' + value; - } - // Normalize option cast.object - if(options.cast.object === undefined || options.cast.object === null){ - // Stringify object as JSON by default - options.cast.object = value => JSON.stringify(value); - } - // Normalize option cast.string - if(options.cast.string === undefined || options.cast.string === null){ - // Leave string untouched - options.cast.string = function(value){return value;}; - } - // Normalize option `record_delimiter` - if(options.record_delimiter === undefined || options.record_delimiter === null){ - options.record_delimiter = '\n'; - }else if(Buffer.isBuffer(options.record_delimiter)){ - options.record_delimiter = options.record_delimiter.toString(); - }else if(typeof options.record_delimiter !== 'string'){ - return Error(`Invalid Option: record_delimiter must be a buffer or a string, got ${JSON.stringify(options.record_delimiter)}`); - } + columns = newcolumns; } - _transform(chunk, encoding, callback){ - if(this.state.stop === true){ - return; - } - const err = this.__transform(chunk); - if(err !== undefined){ - this.state.stop = true; - } - callback(err); + return [undefined, columns]; +}; + +const underscore = function(str){ + return str.replace(/([A-Z])/g, function(_, match){ + return '_' + match.toLowerCase(); + }); +}; + +const normalize_options = function(opts) { + const options = {}; + // Merge with user options + for(const opt in opts){ + options[underscore(opt)] = opts[opt]; } - _flush(callback){ - if(this.state.stop === true){ - // Note, Node.js 12 call flush even after an error, we must prevent - // `callback` from being called in flush without any error. - return; - } - if(this.info.records === 0){ - this.bom(); - const err = this.headers(); - if(err) callback(err); - } - callback(); + // Normalize option `bom` + if(options.bom === undefined || options.bom === null || options.bom === false){ + options.bom = false; + }else if(options.bom !== true){ + return [new CsvError('CSV_OPTION_BOOLEAN_INVALID_TYPE', [ + 'option `bom` is optional and must be a boolean value,', + `got ${JSON.stringify(options.bom)}` + ])]; } - __transform(chunk){ - // Chunk validation - if(!Array.isArray(chunk) && typeof chunk !== 'object'){ - return Error(`Invalid Record: expect an array or an object, got ${JSON.stringify(chunk)}`); - } - // Detect columns from the first record - if(this.info.records === 0){ - if(Array.isArray(chunk)){ - if(this.options.header === true && this.options.columns === undefined){ - return Error('Undiscoverable Columns: header option requires column option or object records'); - } - }else if(this.options.columns === undefined){ - const [err, columns] = this.normalize_columns(Object.keys(chunk)); - if(err) return; - this.options.columns = columns; + // Normalize option `delimiter` + if(options.delimiter === undefined || options.delimiter === null){ + options.delimiter = ','; + }else if(Buffer.isBuffer(options.delimiter)){ + options.delimiter = options.delimiter.toString(); + }else if(typeof options.delimiter !== 'string'){ + return [new CsvError('CSV_OPTION_DELIMITER_INVALID_TYPE', [ + 'option `delimiter` must be a buffer or a string,', + `got ${JSON.stringify(options.delimiter)}` + ])]; + } + // Normalize option `quote` + if(options.quote === undefined || options.quote === null){ + options.quote = '"'; + }else if(options.quote === true){ + options.quote = '"'; + }else if(options.quote === false){ + options.quote = ''; + }else if (Buffer.isBuffer(options.quote)){ + options.quote = options.quote.toString(); + }else if(typeof options.quote !== 'string'){ + return [new CsvError('CSV_OPTION_QUOTE_INVALID_TYPE', [ + 'option `quote` must be a boolean, a buffer or a string,', + `got ${JSON.stringify(options.quote)}` + ])]; + } + // Normalize option `quoted` + if(options.quoted === undefined || options.quoted === null){ + options.quoted = false; + } + // Normalize option `quoted_empty` + if(options.quoted_empty === undefined || options.quoted_empty === null){ + options.quoted_empty = undefined; + } + // Normalize option `quoted_match` + if(options.quoted_match === undefined || options.quoted_match === null || options.quoted_match === false){ + options.quoted_match = null; + }else if(!Array.isArray(options.quoted_match)){ + options.quoted_match = [options.quoted_match]; + } + if(options.quoted_match){ + for(const quoted_match of options.quoted_match){ + const isString = typeof quoted_match === 'string'; + const isRegExp = quoted_match instanceof RegExp; + if(!isString && !isRegExp){ + return [Error(`Invalid Option: quoted_match must be a string or a regex, got ${JSON.stringify(quoted_match)}`)]; } } - // Emit the header - if(this.info.records === 0){ - this.bom(); - const err = this.headers(); - if(err) return err; - } - // Emit and stringify the record if an object or an array - try{ - this.emit('record', chunk, this.info.records); - }catch(err){ - return err; - } - // Convert the record into a string - let err, chunk_string; - if(this.options.eof){ - [err, chunk_string] = this.stringify(chunk); - if(err) return err; - if(chunk_string === undefined){ - return; - }else { - chunk_string = chunk_string + this.options.record_delimiter; + } + // Normalize option `quoted_string` + if(options.quoted_string === undefined || options.quoted_string === null){ + options.quoted_string = false; + } + // Normalize option `eof` + if(options.eof === undefined || options.eof === null){ + options.eof = true; + } + // Normalize option `escape` + if(options.escape === undefined || options.escape === null){ + options.escape = '"'; + }else if(Buffer.isBuffer(options.escape)){ + options.escape = options.escape.toString(); + }else if(typeof options.escape !== 'string'){ + return [Error(`Invalid Option: escape must be a buffer or a string, got ${JSON.stringify(options.escape)}`)]; + } + if (options.escape.length > 1){ + return [Error(`Invalid Option: escape must be one character, got ${options.escape.length} characters`)]; + } + // Normalize option `header` + if(options.header === undefined || options.header === null){ + options.header = false; + } + // Normalize option `columns` + const [errColumns, columns] = normalize_columns(options.columns); + if(errColumns !== undefined) return [errColumns]; + options.columns = columns; + // Normalize option `quoted` + if(options.quoted === undefined || options.quoted === null){ + options.quoted = false; + } + // Normalize option `cast` + if(options.cast === undefined || options.cast === null){ + options.cast = {}; + } + // Normalize option cast.bigint + if(options.cast.bigint === undefined || options.cast.bigint === null){ + // Cast boolean to string by default + options.cast.bigint = value => '' + value; + } + // Normalize option cast.boolean + if(options.cast.boolean === undefined || options.cast.boolean === null){ + // Cast boolean to string by default + options.cast.boolean = value => value ? '1' : ''; + } + // Normalize option cast.date + if(options.cast.date === undefined || options.cast.date === null){ + // Cast date to timestamp string by default + options.cast.date = value => '' + value.getTime(); + } + // Normalize option cast.number + if(options.cast.number === undefined || options.cast.number === null){ + // Cast number to string using native casting by default + options.cast.number = value => '' + value; + } + // Normalize option cast.object + if(options.cast.object === undefined || options.cast.object === null){ + // Stringify object as JSON by default + options.cast.object = value => JSON.stringify(value); + } + // Normalize option cast.string + if(options.cast.string === undefined || options.cast.string === null){ + // Leave string untouched + options.cast.string = function(value){return value;}; + } + // Normalize option `on_record` + if(options.on_record !== undefined && typeof options.on_record !== 'function'){ + return [Error(`Invalid Option: "on_record" must be a function.`)]; + } + // Normalize option `record_delimiter` + if(options.record_delimiter === undefined || options.record_delimiter === null){ + options.record_delimiter = '\n'; + }else if(Buffer.isBuffer(options.record_delimiter)){ + options.record_delimiter = options.record_delimiter.toString(); + }else if(typeof options.record_delimiter !== 'string'){ + return [Error(`Invalid Option: record_delimiter must be a buffer or a string, got ${JSON.stringify(options.record_delimiter)}`)]; + } + switch(options.record_delimiter){ + case 'auto': + options.record_delimiter = null; + break; + case 'unix': + options.record_delimiter = "\n"; + break; + case 'mac': + options.record_delimiter = "\r"; + break; + case 'windows': + options.record_delimiter = "\r\n"; + break; + case 'ascii': + options.record_delimiter = "\u001e"; + break; + case 'unicode': + options.record_delimiter = "\u2028"; + break; + } + return [undefined, options]; +}; + +const bom_utf8 = Buffer.from([239, 187, 191]); + +const stringifier = function(options, state, info){ + return { + options: options, + state: state, + info: info, + __transform: function(chunk, push){ + // Chunk validation + if(!Array.isArray(chunk) && typeof chunk !== 'object'){ + return Error(`Invalid Record: expect an array or an object, got ${JSON.stringify(chunk)}`); } - }else { - [err, chunk_string] = this.stringify(chunk); - if(err) return err; - if(chunk_string === undefined){ - return; - }else { - if(this.options.header || this.info.records){ - chunk_string = this.options.record_delimiter + chunk_string; + // Detect columns from the first record + if(this.info.records === 0){ + if(Array.isArray(chunk)){ + if(this.options.header === true && this.options.columns === undefined){ + return Error('Undiscoverable Columns: header option requires column option or object records'); + } + }else if(this.options.columns === undefined){ + const [err, columns] = normalize_columns(Object.keys(chunk)); + if(err) return; + this.options.columns = columns; } } - } - // Emit the csv - this.info.records++; - this.push(chunk_string); - } - stringify(chunk, chunkIsHeader=false){ - if(typeof chunk !== 'object'){ - return [undefined, chunk]; - } - const {columns} = this.options; - const record = []; - // Record is an array - if(Array.isArray(chunk)){ - // We are getting an array but the user has specified output columns. In - // this case, we respect the columns indexes - if(columns){ - chunk.splice(columns.length); + // Emit the header + if(this.info.records === 0){ + this.bom(push); + const err = this.headers(push); + if(err) return err; } - // Cast record elements - for(let i=0; i= 0; - const containsQuote = (quote !== '') && value.indexOf(quote) >= 0; - const containsEscape = value.indexOf(escape) >= 0 && (escape !== quote); - const containsRecordDelimiter = value.indexOf(record_delimiter) >= 0; - const quotedString = quoted_string && typeof field === 'string'; - let quotedMatch = quoted_match && quoted_match.filter(quoted_match => { - if(typeof quoted_match === 'string'){ - return value.indexOf(quoted_match) !== -1; - }else { - return quoted_match.test(value); + } + let csvrecord = ''; + for(let i=0; i 0; - const shouldQuote = containsQuote === true || containsdelimiter || containsRecordDelimiter || quoted || quotedString || quotedMatch; - if(shouldQuote === true && containsEscape === true){ - const regexp = escape === '\\' - ? new RegExp(escape + escape, 'g') - : new RegExp(escape, 'g'); - value = value.replace(regexp, escape + escape); + options = {...this.options, ...options}; + [err, options] = normalize_options(options); + if(err !== undefined){ + return [err]; + } + }else if(value === undefined || value === null){ + options = this.options; + }else { + return [Error(`Invalid Casting Value: returned value must return a string, an object, null or undefined, got ${JSON.stringify(value)}`)]; } - if(containsQuote === true){ - const regexp = new RegExp(quote,'g'); - value = value.replace(regexp, escape + quote); + const {delimiter, escape, quote, quoted, quoted_empty, quoted_string, quoted_match, record_delimiter} = options; + if(value){ + if(typeof value !== 'string'){ + return [Error(`Formatter must return a string, null or undefined, got ${JSON.stringify(value)}`)]; + } + const containsdelimiter = delimiter.length && value.indexOf(delimiter) >= 0; + const containsQuote = (quote !== '') && value.indexOf(quote) >= 0; + const containsEscape = value.indexOf(escape) >= 0 && (escape !== quote); + const containsRecordDelimiter = value.indexOf(record_delimiter) >= 0; + const quotedString = quoted_string && typeof field === 'string'; + let quotedMatch = quoted_match && quoted_match.filter(quoted_match => { + if(typeof quoted_match === 'string'){ + return value.indexOf(quoted_match) !== -1; + }else { + return quoted_match.test(value); + } + }); + quotedMatch = quotedMatch && quotedMatch.length > 0; + const shouldQuote = containsQuote === true || containsdelimiter || containsRecordDelimiter || quoted || quotedString || quotedMatch; + if(shouldQuote === true && containsEscape === true){ + const regexp = escape === '\\' + ? new RegExp(escape + escape, 'g') + : new RegExp(escape, 'g'); + value = value.replace(regexp, escape + escape); + } + if(containsQuote === true){ + const regexp = new RegExp(quote,'g'); + value = value.replace(regexp, escape + quote); + } + if(shouldQuote === true){ + value = quote + value + quote; + } + csvrecord += value; + }else if(quoted_empty === true || (field === '' && quoted_string === true && quoted_empty !== false)){ + csvrecord += quote + quote; } - if(shouldQuote === true){ - value = quote + value + quote; + if(i !== record.length - 1){ + csvrecord += delimiter; } - csvrecord += value; - }else if(quoted_empty === true || (field === '' && quoted_string === true && quoted_empty !== false)){ - csvrecord += quote + quote; } - if(i !== record.length - 1){ - csvrecord += delimiter; + return [undefined, csvrecord]; + }, + bom: function(push){ + if(this.options.bom !== true){ + return; + } + push(bom_utf8); + }, + headers: function(push){ + if(this.options.header === false){ + return; + } + if(this.options.columns === undefined){ + return; + } + let err; + let headers = this.options.columns.map(column => column.header); + if(this.options.eof){ + [err, headers] = this.stringify(headers, true); + headers += this.options.record_delimiter; + }else { + [err, headers] = this.stringify(headers); + } + if(err) return err; + push(headers); + }, + __cast: function(value, context){ + const type = typeof value; + try{ + if(type === 'string'){ // Fine for 99% of the cases + return [undefined, this.options.cast.string(value, context)]; + }else if(type === 'bigint'){ + return [undefined, this.options.cast.bigint(value, context)]; + }else if(type === 'number'){ + return [undefined, this.options.cast.number(value, context)]; + }else if(type === 'boolean'){ + return [undefined, this.options.cast.boolean(value, context)]; + }else if(value instanceof Date){ + return [undefined, this.options.cast.date(value, context)]; + }else if(type === 'object' && value !== null){ + return [undefined, this.options.cast.object(value, context)]; + }else { + return [undefined, value, value]; + } + }catch(err){ + return [err]; } } - return [undefined, csvrecord]; - } - bom(){ - if(this.options.bom !== true){ - return; - } - this.push(bom_utf8); + }; +}; + +class Stringifier extends stream.Transform { + constructor(opts = {}){ + super({...{writableObjectMode: true}, ...opts}); + const [err, options] = normalize_options(opts); + if(err !== undefined) throw err; + // Expose options + this.options = options; + // Internal state + this.state = { + stop: false + }; + // Information + this.info = { + records: 0 + }; + this.api = stringifier(this.options, this.state, this.info); + this.api.options.on_record = (...args) => { + this.emit('record', ...args); + }; } - headers(){ - if(this.options.header === false){ - return; - } - if(this.options.columns === undefined){ + _transform(chunk, encoding, callback){ + if(this.state.stop === true){ return; } - let err; - let headers = this.options.columns.map(column => column.header); - if(this.options.eof){ - [err, headers] = this.stringify(headers, true); - headers += this.options.record_delimiter; - }else { - [err, headers] = this.stringify(headers); - } - if(err) return err; - this.push(headers); - } - __cast(value, context){ - const type = typeof value; - try{ - if(type === 'string'){ // Fine for 99% of the cases - return [undefined, this.options.cast.string(value, context)]; - }else if(type === 'bigint'){ - return [undefined, this.options.cast.bigint(value, context)]; - }else if(type === 'number'){ - return [undefined, this.options.cast.number(value, context)]; - }else if(type === 'boolean'){ - return [undefined, this.options.cast.boolean(value, context)]; - }else if(value instanceof Date){ - return [undefined, this.options.cast.date(value, context)]; - }else if(type === 'object' && value !== null){ - return [undefined, this.options.cast.object(value, context)]; - }else { - return [undefined, value, value]; - } - }catch(err){ - return [err]; + const err = this.api.__transform(chunk, this.push.bind(this)); + if(err !== undefined){ + this.state.stop = true; } + callback(err); } - normalize_columns(columns){ - if(columns === undefined || columns === null){ - return []; - } - if(typeof columns !== 'object'){ - return [Error('Invalid option "columns": expect an array or an object')]; + _flush(callback){ + if(this.state.stop === true){ + // Note, Node.js 12 call flush even after an error, we must prevent + // `callback` from being called in flush without any error. + return; } - if(!Array.isArray(columns)){ - const newcolumns = []; - for(const k in columns){ - newcolumns.push({ - key: k, - header: columns[k] - }); - } - columns = newcolumns; - }else { - const newcolumns = []; - for(const column of columns){ - if(typeof column === 'string'){ - newcolumns.push({ - key: column, - header: column - }); - }else if(typeof column === 'object' && column !== undefined && !Array.isArray(column)){ - if(!column.key){ - return [Error('Invalid column definition: property "key" is required')]; - } - if(column.header === undefined){ - column.header = column.key; - } - newcolumns.push(column); - }else { - return [Error('Invalid column definition: expect a string or an object')]; - } - } - columns = newcolumns; + if(this.info.records === 0){ + this.api.bom(this.push.bind(this)); + const err = this.api.headers(this.push.bind(this)); + if(err) callback(err); } - return [undefined, columns]; + callback(); } } @@ -555,7 +578,7 @@ const stringify = function(){ const type = typeof argument; if(data === undefined && (Array.isArray(argument))){ data = argument; - }else if(options === undefined && isObject(argument)){ + }else if(options === undefined && is_object(argument)){ options = argument; }else if(callback === undefined && type === 'function'){ callback = argument; diff --git a/packages/csv-stringify/dist/cjs/sync.cjs b/packages/csv-stringify/dist/cjs/sync.cjs index 17adf23dc..c4f067015 100644 --- a/packages/csv-stringify/dist/cjs/sync.cjs +++ b/packages/csv-stringify/dist/cjs/sync.cjs @@ -2,37 +2,6 @@ Object.defineProperty(exports, '__esModule', { value: true }); -var stream = require('stream'); - -const bom_utf8 = Buffer.from([239, 187, 191]); - -class CsvError extends Error { - constructor(code, message, ...contexts) { - if(Array.isArray(message)) message = message.join(' '); - super(message); - if(Error.captureStackTrace !== undefined){ - Error.captureStackTrace(this, CsvError); - } - this.code = code; - for(const context of contexts){ - for(const key in context){ - const value = context[key]; - this[key] = Buffer.isBuffer(value) ? value.toString() : value == null ? value : JSON.parse(JSON.stringify(value)); - } - } - } -} - -const isObject = function(obj){ - return typeof obj === 'object' && obj !== null && ! Array.isArray(obj); -}; - -const underscore = function(str){ - return str.replace(/([A-Z])/g, function(_, match){ - return '_' + match.toLowerCase(); - }); -}; - // Lodash implementation of `get` const charCodeOfDot = '.'.charCodeAt(0); @@ -110,455 +79,473 @@ const get = function(object, path){ return (index && index === length) ? object : undefined; }; -class Stringifier extends stream.Transform { - constructor(opts = {}){ - super({...{writableObjectMode: true}, ...opts}); - const options = {}; - let err; - // Merge with user options - for(const opt in opts){ - options[underscore(opt)] = opts[opt]; - } - if((err = this.normalize(options)) !== undefined) throw err; - switch(options.record_delimiter){ - case 'auto': - options.record_delimiter = null; - break; - case 'unix': - options.record_delimiter = "\n"; - break; - case 'mac': - options.record_delimiter = "\r"; - break; - case 'windows': - options.record_delimiter = "\r\n"; - break; - case 'ascii': - options.record_delimiter = "\u001e"; - break; - case 'unicode': - options.record_delimiter = "\u2028"; - break; - } - // Expose options - this.options = options; - // Internal state - this.state = { - stop: false - }; - // Information - this.info = { - records: 0 - }; - } - normalize(options){ - // Normalize option `bom` - if(options.bom === undefined || options.bom === null || options.bom === false){ - options.bom = false; - }else if(options.bom !== true){ - return new CsvError('CSV_OPTION_BOOLEAN_INVALID_TYPE', [ - 'option `bom` is optional and must be a boolean value,', - `got ${JSON.stringify(options.bom)}` - ]); - } - // Normalize option `delimiter` - if(options.delimiter === undefined || options.delimiter === null){ - options.delimiter = ','; - }else if(Buffer.isBuffer(options.delimiter)){ - options.delimiter = options.delimiter.toString(); - }else if(typeof options.delimiter !== 'string'){ - return new CsvError('CSV_OPTION_DELIMITER_INVALID_TYPE', [ - 'option `delimiter` must be a buffer or a string,', - `got ${JSON.stringify(options.delimiter)}` - ]); - } - // Normalize option `quote` - if(options.quote === undefined || options.quote === null){ - options.quote = '"'; - }else if(options.quote === true){ - options.quote = '"'; - }else if(options.quote === false){ - options.quote = ''; - }else if (Buffer.isBuffer(options.quote)){ - options.quote = options.quote.toString(); - }else if(typeof options.quote !== 'string'){ - return new CsvError('CSV_OPTION_QUOTE_INVALID_TYPE', [ - 'option `quote` must be a boolean, a buffer or a string,', - `got ${JSON.stringify(options.quote)}` - ]); - } - // Normalize option `quoted` - if(options.quoted === undefined || options.quoted === null){ - options.quoted = false; - } - // Normalize option `quoted_empty` - if(options.quoted_empty === undefined || options.quoted_empty === null){ - options.quoted_empty = undefined; - } - // Normalize option `quoted_match` - if(options.quoted_match === undefined || options.quoted_match === null || options.quoted_match === false){ - options.quoted_match = null; - }else if(!Array.isArray(options.quoted_match)){ - options.quoted_match = [options.quoted_match]; - } - if(options.quoted_match){ - for(const quoted_match of options.quoted_match){ - const isString = typeof quoted_match === 'string'; - const isRegExp = quoted_match instanceof RegExp; - if(!isString && !isRegExp){ - return Error(`Invalid Option: quoted_match must be a string or a regex, got ${JSON.stringify(quoted_match)}`); +const is_object = function(obj){ + return typeof obj === 'object' && obj !== null && ! Array.isArray(obj); +}; + +const normalize_columns = function(columns){ + if(columns === undefined || columns === null){ + return [undefined, undefined]; + } + if(typeof columns !== 'object'){ + return [Error('Invalid option "columns": expect an array or an object')]; + } + if(!Array.isArray(columns)){ + const newcolumns = []; + for(const k in columns){ + newcolumns.push({ + key: k, + header: columns[k] + }); + } + columns = newcolumns; + }else { + const newcolumns = []; + for(const column of columns){ + if(typeof column === 'string'){ + newcolumns.push({ + key: column, + header: column + }); + }else if(typeof column === 'object' && column !== null && !Array.isArray(column)){ + if(!column.key){ + return [Error('Invalid column definition: property "key" is required')]; + } + if(column.header === undefined){ + column.header = column.key; } + newcolumns.push(column); + }else { + return [Error('Invalid column definition: expect a string or an object')]; } } - // Normalize option `quoted_string` - if(options.quoted_string === undefined || options.quoted_string === null){ - options.quoted_string = false; - } - // Normalize option `eof` - if(options.eof === undefined || options.eof === null){ - options.eof = true; - } - // Normalize option `escape` - if(options.escape === undefined || options.escape === null){ - options.escape = '"'; - }else if(Buffer.isBuffer(options.escape)){ - options.escape = options.escape.toString(); - }else if(typeof options.escape !== 'string'){ - return Error(`Invalid Option: escape must be a buffer or a string, got ${JSON.stringify(options.escape)}`); - } - if (options.escape.length > 1){ - return Error(`Invalid Option: escape must be one character, got ${options.escape.length} characters`); - } - // Normalize option `header` - if(options.header === undefined || options.header === null){ - options.header = false; - } - // Normalize option `columns` - const [err, columns] = this.normalize_columns(options.columns); - if(err) return err; - options.columns = columns; - // Normalize option `quoted` - if(options.quoted === undefined || options.quoted === null){ - options.quoted = false; - } - // Normalize option `cast` - if(options.cast === undefined || options.cast === null){ - options.cast = {}; - } - // Normalize option cast.bigint - if(options.cast.bigint === undefined || options.cast.bigint === null){ - // Cast boolean to string by default - options.cast.bigint = value => '' + value; - } - // Normalize option cast.boolean - if(options.cast.boolean === undefined || options.cast.boolean === null){ - // Cast boolean to string by default - options.cast.boolean = value => value ? '1' : ''; - } - // Normalize option cast.date - if(options.cast.date === undefined || options.cast.date === null){ - // Cast date to timestamp string by default - options.cast.date = value => '' + value.getTime(); - } - // Normalize option cast.number - if(options.cast.number === undefined || options.cast.number === null){ - // Cast number to string using native casting by default - options.cast.number = value => '' + value; - } - // Normalize option cast.object - if(options.cast.object === undefined || options.cast.object === null){ - // Stringify object as JSON by default - options.cast.object = value => JSON.stringify(value); - } - // Normalize option cast.string - if(options.cast.string === undefined || options.cast.string === null){ - // Leave string untouched - options.cast.string = function(value){return value;}; - } - // Normalize option `record_delimiter` - if(options.record_delimiter === undefined || options.record_delimiter === null){ - options.record_delimiter = '\n'; - }else if(Buffer.isBuffer(options.record_delimiter)){ - options.record_delimiter = options.record_delimiter.toString(); - }else if(typeof options.record_delimiter !== 'string'){ - return Error(`Invalid Option: record_delimiter must be a buffer or a string, got ${JSON.stringify(options.record_delimiter)}`); - } + columns = newcolumns; } - _transform(chunk, encoding, callback){ - if(this.state.stop === true){ - return; + return [undefined, columns]; +}; + +class CsvError extends Error { + constructor(code, message, ...contexts) { + if(Array.isArray(message)) message = message.join(' '); + super(message); + if(Error.captureStackTrace !== undefined){ + Error.captureStackTrace(this, CsvError); } - const err = this.__transform(chunk); - if(err !== undefined){ - this.state.stop = true; + this.code = code; + for(const context of contexts){ + for(const key in context){ + const value = context[key]; + this[key] = Buffer.isBuffer(value) ? value.toString() : value == null ? value : JSON.parse(JSON.stringify(value)); + } } - callback(err); } - _flush(callback){ - if(this.state.stop === true){ - // Note, Node.js 12 call flush even after an error, we must prevent - // `callback` from being called in flush without any error. - return; - } - if(this.info.records === 0){ - this.bom(); - const err = this.headers(); - if(err) callback(err); - } - callback(); +} + +const underscore = function(str){ + return str.replace(/([A-Z])/g, function(_, match){ + return '_' + match.toLowerCase(); + }); +}; + +const normalize_options = function(opts) { + const options = {}; + // Merge with user options + for(const opt in opts){ + options[underscore(opt)] = opts[opt]; } - __transform(chunk){ - // Chunk validation - if(!Array.isArray(chunk) && typeof chunk !== 'object'){ - return Error(`Invalid Record: expect an array or an object, got ${JSON.stringify(chunk)}`); - } - // Detect columns from the first record - if(this.info.records === 0){ - if(Array.isArray(chunk)){ - if(this.options.header === true && this.options.columns === undefined){ - return Error('Undiscoverable Columns: header option requires column option or object records'); - } - }else if(this.options.columns === undefined){ - const [err, columns] = this.normalize_columns(Object.keys(chunk)); - if(err) return; - this.options.columns = columns; + // Normalize option `bom` + if(options.bom === undefined || options.bom === null || options.bom === false){ + options.bom = false; + }else if(options.bom !== true){ + return [new CsvError('CSV_OPTION_BOOLEAN_INVALID_TYPE', [ + 'option `bom` is optional and must be a boolean value,', + `got ${JSON.stringify(options.bom)}` + ])]; + } + // Normalize option `delimiter` + if(options.delimiter === undefined || options.delimiter === null){ + options.delimiter = ','; + }else if(Buffer.isBuffer(options.delimiter)){ + options.delimiter = options.delimiter.toString(); + }else if(typeof options.delimiter !== 'string'){ + return [new CsvError('CSV_OPTION_DELIMITER_INVALID_TYPE', [ + 'option `delimiter` must be a buffer or a string,', + `got ${JSON.stringify(options.delimiter)}` + ])]; + } + // Normalize option `quote` + if(options.quote === undefined || options.quote === null){ + options.quote = '"'; + }else if(options.quote === true){ + options.quote = '"'; + }else if(options.quote === false){ + options.quote = ''; + }else if (Buffer.isBuffer(options.quote)){ + options.quote = options.quote.toString(); + }else if(typeof options.quote !== 'string'){ + return [new CsvError('CSV_OPTION_QUOTE_INVALID_TYPE', [ + 'option `quote` must be a boolean, a buffer or a string,', + `got ${JSON.stringify(options.quote)}` + ])]; + } + // Normalize option `quoted` + if(options.quoted === undefined || options.quoted === null){ + options.quoted = false; + } + // Normalize option `quoted_empty` + if(options.quoted_empty === undefined || options.quoted_empty === null){ + options.quoted_empty = undefined; + } + // Normalize option `quoted_match` + if(options.quoted_match === undefined || options.quoted_match === null || options.quoted_match === false){ + options.quoted_match = null; + }else if(!Array.isArray(options.quoted_match)){ + options.quoted_match = [options.quoted_match]; + } + if(options.quoted_match){ + for(const quoted_match of options.quoted_match){ + const isString = typeof quoted_match === 'string'; + const isRegExp = quoted_match instanceof RegExp; + if(!isString && !isRegExp){ + return [Error(`Invalid Option: quoted_match must be a string or a regex, got ${JSON.stringify(quoted_match)}`)]; } } - // Emit the header - if(this.info.records === 0){ - this.bom(); - const err = this.headers(); - if(err) return err; - } - // Emit and stringify the record if an object or an array - try{ - this.emit('record', chunk, this.info.records); - }catch(err){ - return err; - } - // Convert the record into a string - let err, chunk_string; - if(this.options.eof){ - [err, chunk_string] = this.stringify(chunk); - if(err) return err; - if(chunk_string === undefined){ - return; - }else { - chunk_string = chunk_string + this.options.record_delimiter; + } + // Normalize option `quoted_string` + if(options.quoted_string === undefined || options.quoted_string === null){ + options.quoted_string = false; + } + // Normalize option `eof` + if(options.eof === undefined || options.eof === null){ + options.eof = true; + } + // Normalize option `escape` + if(options.escape === undefined || options.escape === null){ + options.escape = '"'; + }else if(Buffer.isBuffer(options.escape)){ + options.escape = options.escape.toString(); + }else if(typeof options.escape !== 'string'){ + return [Error(`Invalid Option: escape must be a buffer or a string, got ${JSON.stringify(options.escape)}`)]; + } + if (options.escape.length > 1){ + return [Error(`Invalid Option: escape must be one character, got ${options.escape.length} characters`)]; + } + // Normalize option `header` + if(options.header === undefined || options.header === null){ + options.header = false; + } + // Normalize option `columns` + const [errColumns, columns] = normalize_columns(options.columns); + if(errColumns !== undefined) return [errColumns]; + options.columns = columns; + // Normalize option `quoted` + if(options.quoted === undefined || options.quoted === null){ + options.quoted = false; + } + // Normalize option `cast` + if(options.cast === undefined || options.cast === null){ + options.cast = {}; + } + // Normalize option cast.bigint + if(options.cast.bigint === undefined || options.cast.bigint === null){ + // Cast boolean to string by default + options.cast.bigint = value => '' + value; + } + // Normalize option cast.boolean + if(options.cast.boolean === undefined || options.cast.boolean === null){ + // Cast boolean to string by default + options.cast.boolean = value => value ? '1' : ''; + } + // Normalize option cast.date + if(options.cast.date === undefined || options.cast.date === null){ + // Cast date to timestamp string by default + options.cast.date = value => '' + value.getTime(); + } + // Normalize option cast.number + if(options.cast.number === undefined || options.cast.number === null){ + // Cast number to string using native casting by default + options.cast.number = value => '' + value; + } + // Normalize option cast.object + if(options.cast.object === undefined || options.cast.object === null){ + // Stringify object as JSON by default + options.cast.object = value => JSON.stringify(value); + } + // Normalize option cast.string + if(options.cast.string === undefined || options.cast.string === null){ + // Leave string untouched + options.cast.string = function(value){return value;}; + } + // Normalize option `on_record` + if(options.on_record !== undefined && typeof options.on_record !== 'function'){ + return [Error(`Invalid Option: "on_record" must be a function.`)]; + } + // Normalize option `record_delimiter` + if(options.record_delimiter === undefined || options.record_delimiter === null){ + options.record_delimiter = '\n'; + }else if(Buffer.isBuffer(options.record_delimiter)){ + options.record_delimiter = options.record_delimiter.toString(); + }else if(typeof options.record_delimiter !== 'string'){ + return [Error(`Invalid Option: record_delimiter must be a buffer or a string, got ${JSON.stringify(options.record_delimiter)}`)]; + } + switch(options.record_delimiter){ + case 'auto': + options.record_delimiter = null; + break; + case 'unix': + options.record_delimiter = "\n"; + break; + case 'mac': + options.record_delimiter = "\r"; + break; + case 'windows': + options.record_delimiter = "\r\n"; + break; + case 'ascii': + options.record_delimiter = "\u001e"; + break; + case 'unicode': + options.record_delimiter = "\u2028"; + break; + } + return [undefined, options]; +}; + +const bom_utf8 = Buffer.from([239, 187, 191]); + +const stringifier = function(options, state, info){ + return { + options: options, + state: state, + info: info, + __transform: function(chunk, push){ + // Chunk validation + if(!Array.isArray(chunk) && typeof chunk !== 'object'){ + return Error(`Invalid Record: expect an array or an object, got ${JSON.stringify(chunk)}`); } - }else { - [err, chunk_string] = this.stringify(chunk); - if(err) return err; - if(chunk_string === undefined){ - return; - }else { - if(this.options.header || this.info.records){ - chunk_string = this.options.record_delimiter + chunk_string; + // Detect columns from the first record + if(this.info.records === 0){ + if(Array.isArray(chunk)){ + if(this.options.header === true && this.options.columns === undefined){ + return Error('Undiscoverable Columns: header option requires column option or object records'); + } + }else if(this.options.columns === undefined){ + const [err, columns] = normalize_columns(Object.keys(chunk)); + if(err) return; + this.options.columns = columns; } } - } - // Emit the csv - this.info.records++; - this.push(chunk_string); - } - stringify(chunk, chunkIsHeader=false){ - if(typeof chunk !== 'object'){ - return [undefined, chunk]; - } - const {columns} = this.options; - const record = []; - // Record is an array - if(Array.isArray(chunk)){ - // We are getting an array but the user has specified output columns. In - // this case, we respect the columns indexes - if(columns){ - chunk.splice(columns.length); + // Emit the header + if(this.info.records === 0){ + this.bom(push); + const err = this.headers(push); + if(err) return err; } - // Cast record elements - for(let i=0; i= 0; - const containsQuote = (quote !== '') && value.indexOf(quote) >= 0; - const containsEscape = value.indexOf(escape) >= 0 && (escape !== quote); - const containsRecordDelimiter = value.indexOf(record_delimiter) >= 0; - const quotedString = quoted_string && typeof field === 'string'; - let quotedMatch = quoted_match && quoted_match.filter(quoted_match => { - if(typeof quoted_match === 'string'){ - return value.indexOf(quoted_match) !== -1; - }else { - return quoted_match.test(value); + } + let csvrecord = ''; + for(let i=0; i 0; - const shouldQuote = containsQuote === true || containsdelimiter || containsRecordDelimiter || quoted || quotedString || quotedMatch; - if(shouldQuote === true && containsEscape === true){ - const regexp = escape === '\\' - ? new RegExp(escape + escape, 'g') - : new RegExp(escape, 'g'); - value = value.replace(regexp, escape + escape); + options = {...this.options, ...options}; + [err, options] = normalize_options(options); + if(err !== undefined){ + return [err]; + } + }else if(value === undefined || value === null){ + options = this.options; + }else { + return [Error(`Invalid Casting Value: returned value must return a string, an object, null or undefined, got ${JSON.stringify(value)}`)]; } - if(containsQuote === true){ - const regexp = new RegExp(quote,'g'); - value = value.replace(regexp, escape + quote); + const {delimiter, escape, quote, quoted, quoted_empty, quoted_string, quoted_match, record_delimiter} = options; + if(value){ + if(typeof value !== 'string'){ + return [Error(`Formatter must return a string, null or undefined, got ${JSON.stringify(value)}`)]; + } + const containsdelimiter = delimiter.length && value.indexOf(delimiter) >= 0; + const containsQuote = (quote !== '') && value.indexOf(quote) >= 0; + const containsEscape = value.indexOf(escape) >= 0 && (escape !== quote); + const containsRecordDelimiter = value.indexOf(record_delimiter) >= 0; + const quotedString = quoted_string && typeof field === 'string'; + let quotedMatch = quoted_match && quoted_match.filter(quoted_match => { + if(typeof quoted_match === 'string'){ + return value.indexOf(quoted_match) !== -1; + }else { + return quoted_match.test(value); + } + }); + quotedMatch = quotedMatch && quotedMatch.length > 0; + const shouldQuote = containsQuote === true || containsdelimiter || containsRecordDelimiter || quoted || quotedString || quotedMatch; + if(shouldQuote === true && containsEscape === true){ + const regexp = escape === '\\' + ? new RegExp(escape + escape, 'g') + : new RegExp(escape, 'g'); + value = value.replace(regexp, escape + escape); + } + if(containsQuote === true){ + const regexp = new RegExp(quote,'g'); + value = value.replace(regexp, escape + quote); + } + if(shouldQuote === true){ + value = quote + value + quote; + } + csvrecord += value; + }else if(quoted_empty === true || (field === '' && quoted_string === true && quoted_empty !== false)){ + csvrecord += quote + quote; } - if(shouldQuote === true){ - value = quote + value + quote; + if(i !== record.length - 1){ + csvrecord += delimiter; } - csvrecord += value; - }else if(quoted_empty === true || (field === '' && quoted_string === true && quoted_empty !== false)){ - csvrecord += quote + quote; } - if(i !== record.length - 1){ - csvrecord += delimiter; + return [undefined, csvrecord]; + }, + bom: function(push){ + if(this.options.bom !== true){ + return; } - } - return [undefined, csvrecord]; - } - bom(){ - if(this.options.bom !== true){ - return; - } - this.push(bom_utf8); - } - headers(){ - if(this.options.header === false){ - return; - } - if(this.options.columns === undefined){ - return; - } - let err; - let headers = this.options.columns.map(column => column.header); - if(this.options.eof){ - [err, headers] = this.stringify(headers, true); - headers += this.options.record_delimiter; - }else { - [err, headers] = this.stringify(headers); - } - if(err) return err; - this.push(headers); - } - __cast(value, context){ - const type = typeof value; - try{ - if(type === 'string'){ // Fine for 99% of the cases - return [undefined, this.options.cast.string(value, context)]; - }else if(type === 'bigint'){ - return [undefined, this.options.cast.bigint(value, context)]; - }else if(type === 'number'){ - return [undefined, this.options.cast.number(value, context)]; - }else if(type === 'boolean'){ - return [undefined, this.options.cast.boolean(value, context)]; - }else if(value instanceof Date){ - return [undefined, this.options.cast.date(value, context)]; - }else if(type === 'object' && value !== null){ - return [undefined, this.options.cast.object(value, context)]; - }else { - return [undefined, value, value]; + push(bom_utf8); + }, + headers: function(push){ + if(this.options.header === false){ + return; } - }catch(err){ - return [err]; - } - } - normalize_columns(columns){ - if(columns === undefined || columns === null){ - return []; - } - if(typeof columns !== 'object'){ - return [Error('Invalid option "columns": expect an array or an object')]; - } - if(!Array.isArray(columns)){ - const newcolumns = []; - for(const k in columns){ - newcolumns.push({ - key: k, - header: columns[k] - }); + if(this.options.columns === undefined){ + return; } - columns = newcolumns; - }else { - const newcolumns = []; - for(const column of columns){ - if(typeof column === 'string'){ - newcolumns.push({ - key: column, - header: column - }); - }else if(typeof column === 'object' && column !== undefined && !Array.isArray(column)){ - if(!column.key){ - return [Error('Invalid column definition: property "key" is required')]; - } - if(column.header === undefined){ - column.header = column.key; - } - newcolumns.push(column); + let err; + let headers = this.options.columns.map(column => column.header); + if(this.options.eof){ + [err, headers] = this.stringify(headers, true); + headers += this.options.record_delimiter; + }else { + [err, headers] = this.stringify(headers); + } + if(err) return err; + push(headers); + }, + __cast: function(value, context){ + const type = typeof value; + try{ + if(type === 'string'){ // Fine for 99% of the cases + return [undefined, this.options.cast.string(value, context)]; + }else if(type === 'bigint'){ + return [undefined, this.options.cast.bigint(value, context)]; + }else if(type === 'number'){ + return [undefined, this.options.cast.number(value, context)]; + }else if(type === 'boolean'){ + return [undefined, this.options.cast.boolean(value, context)]; + }else if(value instanceof Date){ + return [undefined, this.options.cast.date(value, context)]; + }else if(type === 'object' && value !== null){ + return [undefined, this.options.cast.object(value, context)]; }else { - return [Error('Invalid column definition: expect a string or an object')]; + return [undefined, value, value]; } + }catch(err){ + return [err]; } - columns = newcolumns; } - return [undefined, columns]; - } -} + }; +}; -const stringify = function(records, options={}){ +const stringify = function(records, opts={}){ const data = []; - const stringifier = new Stringifier(options); - stringifier.push = function(record){ - if(record === null){ - return; - } - data.push(record.toString()); + const [err, options] = normalize_options(opts); + if(err !== undefined) throw err; + const state = { + stop: false + }; + // Information + const info = { + records: 0 }; + const api = stringifier(options, state, info); + // stringifier.push = function(record){ + // if(record === null){ + // return; + // } + // data.push(record.toString()); + // }; for(const record of records){ - const err = stringifier.__transform(record, null); + const err = api.__transform(record, function(record){ + data.push(record); + }); if(err !== undefined) throw err; } return data.join(''); diff --git a/packages/csv-stringify/dist/esm/index.js b/packages/csv-stringify/dist/esm/index.js index 46e8b2a8a..7992dafef 100644 --- a/packages/csv-stringify/dist/esm/index.js +++ b/packages/csv-stringify/dist/esm/index.js @@ -1,2432 +1,2432 @@ -var global$1 = (typeof global !== "undefined" ? global : - typeof self !== "undefined" ? self : - typeof window !== "undefined" ? window : {}); +var domain; -var lookup = []; -var revLookup = []; -var Arr = typeof Uint8Array !== 'undefined' ? Uint8Array : Array; -var inited = false; -function init () { - inited = true; - var code = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'; - for (var i = 0, len = code.length; i < len; ++i) { - lookup[i] = code[i]; - revLookup[code.charCodeAt(i)] = i; - } +// This constructor is used to store event handlers. Instantiating this is +// faster than explicitly calling `Object.create(null)` to get a "clean" empty +// object (tested with v8 v4.9). +function EventHandlers() {} +EventHandlers.prototype = Object.create(null); - revLookup['-'.charCodeAt(0)] = 62; - revLookup['_'.charCodeAt(0)] = 63; +function EventEmitter() { + EventEmitter.init.call(this); } -function toByteArray (b64) { - if (!inited) { - init(); - } - var i, j, l, tmp, placeHolders, arr; - var len = b64.length; - - if (len % 4 > 0) { - throw new Error('Invalid string. Length must be a multiple of 4') - } - - // the number of equal signs (place holders) - // if there are two placeholders, than the two characters before it - // represent one byte - // if there is only one, then the three characters before it represent 2 bytes - // this is just a cheap hack to not do indexOf twice - placeHolders = b64[len - 2] === '=' ? 2 : b64[len - 1] === '=' ? 1 : 0; +// nodejs oddity +// require('events') === require('events').EventEmitter +EventEmitter.EventEmitter = EventEmitter; - // base64 is 4/3 + up to two characters of the original data - arr = new Arr(len * 3 / 4 - placeHolders); +EventEmitter.usingDomains = false; - // if there are placeholders, only get up to the last complete 4 chars - l = placeHolders > 0 ? len - 4 : len; +EventEmitter.prototype.domain = undefined; +EventEmitter.prototype._events = undefined; +EventEmitter.prototype._maxListeners = undefined; - var L = 0; +// By default EventEmitters will print a warning if more than 10 listeners are +// added to it. This is a useful default which helps finding memory leaks. +EventEmitter.defaultMaxListeners = 10; - for (i = 0, j = 0; i < l; i += 4, j += 3) { - tmp = (revLookup[b64.charCodeAt(i)] << 18) | (revLookup[b64.charCodeAt(i + 1)] << 12) | (revLookup[b64.charCodeAt(i + 2)] << 6) | revLookup[b64.charCodeAt(i + 3)]; - arr[L++] = (tmp >> 16) & 0xFF; - arr[L++] = (tmp >> 8) & 0xFF; - arr[L++] = tmp & 0xFF; +EventEmitter.init = function() { + this.domain = null; + if (EventEmitter.usingDomains) { + // if there is an active domain, then attach to it. + if (domain.active ) ; } - if (placeHolders === 2) { - tmp = (revLookup[b64.charCodeAt(i)] << 2) | (revLookup[b64.charCodeAt(i + 1)] >> 4); - arr[L++] = tmp & 0xFF; - } else if (placeHolders === 1) { - tmp = (revLookup[b64.charCodeAt(i)] << 10) | (revLookup[b64.charCodeAt(i + 1)] << 4) | (revLookup[b64.charCodeAt(i + 2)] >> 2); - arr[L++] = (tmp >> 8) & 0xFF; - arr[L++] = tmp & 0xFF; + if (!this._events || this._events === Object.getPrototypeOf(this)._events) { + this._events = new EventHandlers(); + this._eventsCount = 0; } - return arr -} + this._maxListeners = this._maxListeners || undefined; +}; -function tripletToBase64 (num) { - return lookup[num >> 18 & 0x3F] + lookup[num >> 12 & 0x3F] + lookup[num >> 6 & 0x3F] + lookup[num & 0x3F] +// Obviously not all Emitters should be limited to 10. This function allows +// that to be increased. Set to zero for unlimited. +EventEmitter.prototype.setMaxListeners = function setMaxListeners(n) { + if (typeof n !== 'number' || n < 0 || isNaN(n)) + throw new TypeError('"n" argument must be a positive number'); + this._maxListeners = n; + return this; +}; + +function $getMaxListeners(that) { + if (that._maxListeners === undefined) + return EventEmitter.defaultMaxListeners; + return that._maxListeners; } -function encodeChunk (uint8, start, end) { - var tmp; - var output = []; - for (var i = start; i < end; i += 3) { - tmp = (uint8[i] << 16) + (uint8[i + 1] << 8) + (uint8[i + 2]); - output.push(tripletToBase64(tmp)); +EventEmitter.prototype.getMaxListeners = function getMaxListeners() { + return $getMaxListeners(this); +}; + +// These standalone emit* functions are used to optimize calling of event +// handlers for fast cases because emit() itself often has a variable number of +// arguments and can be deoptimized because of that. These functions always have +// the same number of arguments and thus do not get deoptimized, so the code +// inside them can execute faster. +function emitNone(handler, isFn, self) { + if (isFn) + handler.call(self); + else { + var len = handler.length; + var listeners = arrayClone(handler, len); + for (var i = 0; i < len; ++i) + listeners[i].call(self); } - return output.join('') } - -function fromByteArray (uint8) { - if (!inited) { - init(); +function emitOne(handler, isFn, self, arg1) { + if (isFn) + handler.call(self, arg1); + else { + var len = handler.length; + var listeners = arrayClone(handler, len); + for (var i = 0; i < len; ++i) + listeners[i].call(self, arg1); } - var tmp; - var len = uint8.length; - var extraBytes = len % 3; // if we have 1 byte left, pad 2 bytes - var output = ''; - var parts = []; - var maxChunkLength = 16383; // must be multiple of 3 - - // go through the array every three bytes, we'll deal with trailing stuff later - for (var i = 0, len2 = len - extraBytes; i < len2; i += maxChunkLength) { - parts.push(encodeChunk(uint8, i, (i + maxChunkLength) > len2 ? len2 : (i + maxChunkLength))); +} +function emitTwo(handler, isFn, self, arg1, arg2) { + if (isFn) + handler.call(self, arg1, arg2); + else { + var len = handler.length; + var listeners = arrayClone(handler, len); + for (var i = 0; i < len; ++i) + listeners[i].call(self, arg1, arg2); } - - // pad the end with zeros, but make sure to not forget the extra bytes - if (extraBytes === 1) { - tmp = uint8[len - 1]; - output += lookup[tmp >> 2]; - output += lookup[(tmp << 4) & 0x3F]; - output += '=='; - } else if (extraBytes === 2) { - tmp = (uint8[len - 2] << 8) + (uint8[len - 1]); - output += lookup[tmp >> 10]; - output += lookup[(tmp >> 4) & 0x3F]; - output += lookup[(tmp << 2) & 0x3F]; - output += '='; +} +function emitThree(handler, isFn, self, arg1, arg2, arg3) { + if (isFn) + handler.call(self, arg1, arg2, arg3); + else { + var len = handler.length; + var listeners = arrayClone(handler, len); + for (var i = 0; i < len; ++i) + listeners[i].call(self, arg1, arg2, arg3); } - - parts.push(output); - - return parts.join('') } -function read (buffer, offset, isLE, mLen, nBytes) { - var e, m; - var eLen = nBytes * 8 - mLen - 1; - var eMax = (1 << eLen) - 1; - var eBias = eMax >> 1; - var nBits = -7; - var i = isLE ? (nBytes - 1) : 0; - var d = isLE ? -1 : 1; - var s = buffer[offset + i]; +function emitMany(handler, isFn, self, args) { + if (isFn) + handler.apply(self, args); + else { + var len = handler.length; + var listeners = arrayClone(handler, len); + for (var i = 0; i < len; ++i) + listeners[i].apply(self, args); + } +} - i += d; +EventEmitter.prototype.emit = function emit(type) { + var er, handler, len, args, i, events, domain; + var doError = (type === 'error'); - e = s & ((1 << (-nBits)) - 1); - s >>= (-nBits); - nBits += eLen; - for (; nBits > 0; e = e * 256 + buffer[offset + i], i += d, nBits -= 8) {} + events = this._events; + if (events) + doError = (doError && events.error == null); + else if (!doError) + return false; - m = e & ((1 << (-nBits)) - 1); - e >>= (-nBits); - nBits += mLen; - for (; nBits > 0; m = m * 256 + buffer[offset + i], i += d, nBits -= 8) {} + domain = this.domain; - if (e === 0) { - e = 1 - eBias; - } else if (e === eMax) { - return m ? NaN : ((s ? -1 : 1) * Infinity) - } else { - m = m + Math.pow(2, mLen); - e = e - eBias; + // If there is no 'error' event listener then throw. + if (doError) { + er = arguments[1]; + if (domain) { + if (!er) + er = new Error('Uncaught, unspecified "error" event'); + er.domainEmitter = this; + er.domain = domain; + er.domainThrown = false; + domain.emit('error', er); + } else if (er instanceof Error) { + throw er; // Unhandled 'error' event + } else { + // At least give some kind of context to the user + var err = new Error('Uncaught, unspecified "error" event. (' + er + ')'); + err.context = er; + throw err; + } + return false; } - return (s ? -1 : 1) * m * Math.pow(2, e - mLen) -} -function write (buffer, value, offset, isLE, mLen, nBytes) { - var e, m, c; - var eLen = nBytes * 8 - mLen - 1; - var eMax = (1 << eLen) - 1; - var eBias = eMax >> 1; - var rt = (mLen === 23 ? Math.pow(2, -24) - Math.pow(2, -77) : 0); - var i = isLE ? 0 : (nBytes - 1); - var d = isLE ? 1 : -1; - var s = value < 0 || (value === 0 && 1 / value < 0) ? 1 : 0; + handler = events[type]; - value = Math.abs(value); + if (!handler) + return false; - if (isNaN(value) || value === Infinity) { - m = isNaN(value) ? 1 : 0; - e = eMax; - } else { - e = Math.floor(Math.log(value) / Math.LN2); - if (value * (c = Math.pow(2, -e)) < 1) { - e--; - c *= 2; - } - if (e + eBias >= 1) { - value += rt / c; - } else { - value += rt * Math.pow(2, 1 - eBias); - } - if (value * c >= 2) { - e++; - c /= 2; - } - - if (e + eBias >= eMax) { - m = 0; - e = eMax; - } else if (e + eBias >= 1) { - m = (value * c - 1) * Math.pow(2, mLen); - e = e + eBias; - } else { - m = value * Math.pow(2, eBias - 1) * Math.pow(2, mLen); - e = 0; - } + var isFn = typeof handler === 'function'; + len = arguments.length; + switch (len) { + // fast cases + case 1: + emitNone(handler, isFn, this); + break; + case 2: + emitOne(handler, isFn, this, arguments[1]); + break; + case 3: + emitTwo(handler, isFn, this, arguments[1], arguments[2]); + break; + case 4: + emitThree(handler, isFn, this, arguments[1], arguments[2], arguments[3]); + break; + // slower + default: + args = new Array(len - 1); + for (i = 1; i < len; i++) + args[i - 1] = arguments[i]; + emitMany(handler, isFn, this, args); } - for (; mLen >= 8; buffer[offset + i] = m & 0xff, i += d, m /= 256, mLen -= 8) {} - - e = (e << mLen) | m; - eLen += mLen; - for (; eLen > 0; buffer[offset + i] = e & 0xff, i += d, e /= 256, eLen -= 8) {} - - buffer[offset + i - d] |= s * 128; -} - -var toString = {}.toString; - -var isArray$1 = Array.isArray || function (arr) { - return toString.call(arr) == '[object Array]'; + return true; }; -var INSPECT_MAX_BYTES = 50; - -/** - * If `Buffer.TYPED_ARRAY_SUPPORT`: - * === true Use Uint8Array implementation (fastest) - * === false Use Object implementation (most compatible, even IE6) - * - * Browsers that support typed arrays are IE 10+, Firefox 4+, Chrome 7+, Safari 5.1+, - * Opera 11.6+, iOS 4.2+. - * - * Due to various browser bugs, sometimes the Object implementation will be used even - * when the browser supports typed arrays. - * - * Note: - * - * - Firefox 4-29 lacks support for adding new properties to `Uint8Array` instances, - * See: https://bugzilla.mozilla.org/show_bug.cgi?id=695438. - * - * - Chrome 9-10 is missing the `TypedArray.prototype.subarray` function. - * - * - IE10 has a broken `TypedArray.prototype.subarray` function which returns arrays of - * incorrect length in some situations. +function _addListener(target, type, listener, prepend) { + var m; + var events; + var existing; - * We detect these buggy browsers and set `Buffer.TYPED_ARRAY_SUPPORT` to `false` so they - * get the Object implementation, which is slower but behaves correctly. - */ -Buffer.TYPED_ARRAY_SUPPORT = global$1.TYPED_ARRAY_SUPPORT !== undefined - ? global$1.TYPED_ARRAY_SUPPORT - : true; + if (typeof listener !== 'function') + throw new TypeError('"listener" argument must be a function'); -function kMaxLength () { - return Buffer.TYPED_ARRAY_SUPPORT - ? 0x7fffffff - : 0x3fffffff -} + events = target._events; + if (!events) { + events = target._events = new EventHandlers(); + target._eventsCount = 0; + } else { + // To avoid recursion in the case that type === "newListener"! Before + // adding it to the listeners, first emit "newListener". + if (events.newListener) { + target.emit('newListener', type, + listener.listener ? listener.listener : listener); -function createBuffer (that, length) { - if (kMaxLength() < length) { - throw new RangeError('Invalid typed array length') + // Re-assign `events` because a newListener handler could have caused the + // this._events to be assigned to a new object + events = target._events; + } + existing = events[type]; } - if (Buffer.TYPED_ARRAY_SUPPORT) { - // Return an augmented `Uint8Array` instance, for best performance - that = new Uint8Array(length); - that.__proto__ = Buffer.prototype; + + if (!existing) { + // Optimize the case of one listener. Don't need the extra array object. + existing = events[type] = listener; + ++target._eventsCount; } else { - // Fallback: Return an object instance of the Buffer class - if (that === null) { - that = new Buffer(length); + if (typeof existing === 'function') { + // Adding the second element, need to change to array. + existing = events[type] = prepend ? [listener, existing] : + [existing, listener]; + } else { + // If we've already got an array, just append. + if (prepend) { + existing.unshift(listener); + } else { + existing.push(listener); + } + } + + // Check for listener leak + if (!existing.warned) { + m = $getMaxListeners(target); + if (m && m > 0 && existing.length > m) { + existing.warned = true; + var w = new Error('Possible EventEmitter memory leak detected. ' + + existing.length + ' ' + type + ' listeners added. ' + + 'Use emitter.setMaxListeners() to increase limit'); + w.name = 'MaxListenersExceededWarning'; + w.emitter = target; + w.type = type; + w.count = existing.length; + emitWarning(w); + } } - that.length = length; } - return that + return target; +} +function emitWarning(e) { + typeof console.warn === 'function' ? console.warn(e) : console.log(e); } +EventEmitter.prototype.addListener = function addListener(type, listener) { + return _addListener(this, type, listener, false); +}; -/** - * The Buffer constructor returns instances of `Uint8Array` that have their - * prototype changed to `Buffer.prototype`. Furthermore, `Buffer` is a subclass of - * `Uint8Array`, so the returned instances will have all the node `Buffer` methods - * and the `Uint8Array` methods. Square bracket notation works as expected -- it - * returns a single octet. - * - * The `Uint8Array` prototype remains unmodified. - */ +EventEmitter.prototype.on = EventEmitter.prototype.addListener; -function Buffer (arg, encodingOrOffset, length) { - if (!Buffer.TYPED_ARRAY_SUPPORT && !(this instanceof Buffer)) { - return new Buffer(arg, encodingOrOffset, length) - } +EventEmitter.prototype.prependListener = + function prependListener(type, listener) { + return _addListener(this, type, listener, true); + }; - // Common case. - if (typeof arg === 'number') { - if (typeof encodingOrOffset === 'string') { - throw new Error( - 'If encoding is specified then the first argument must be a string' - ) +function _onceWrap(target, type, listener) { + var fired = false; + function g() { + target.removeListener(type, g); + if (!fired) { + fired = true; + listener.apply(target, arguments); } - return allocUnsafe(this, arg) } - return from(this, arg, encodingOrOffset, length) + g.listener = listener; + return g; } -Buffer.poolSize = 8192; // not used by this implementation - -// TODO: Legacy, not needed anymore. Remove in next major version. -Buffer._augment = function (arr) { - arr.__proto__ = Buffer.prototype; - return arr +EventEmitter.prototype.once = function once(type, listener) { + if (typeof listener !== 'function') + throw new TypeError('"listener" argument must be a function'); + this.on(type, _onceWrap(this, type, listener)); + return this; }; -function from (that, value, encodingOrOffset, length) { - if (typeof value === 'number') { - throw new TypeError('"value" argument must not be a number') - } +EventEmitter.prototype.prependOnceListener = + function prependOnceListener(type, listener) { + if (typeof listener !== 'function') + throw new TypeError('"listener" argument must be a function'); + this.prependListener(type, _onceWrap(this, type, listener)); + return this; + }; - if (typeof ArrayBuffer !== 'undefined' && value instanceof ArrayBuffer) { - return fromArrayBuffer(that, value, encodingOrOffset, length) - } +// emits a 'removeListener' event iff the listener was removed +EventEmitter.prototype.removeListener = + function removeListener(type, listener) { + var list, events, position, i, originalListener; - if (typeof value === 'string') { - return fromString(that, value, encodingOrOffset) - } + if (typeof listener !== 'function') + throw new TypeError('"listener" argument must be a function'); - return fromObject(that, value) -} + events = this._events; + if (!events) + return this; -/** - * Functionally equivalent to Buffer(arg, encoding) but throws a TypeError - * if value is a number. - * Buffer.from(str[, encoding]) - * Buffer.from(array) - * Buffer.from(buffer) - * Buffer.from(arrayBuffer[, byteOffset[, length]]) - **/ -Buffer.from = function (value, encodingOrOffset, length) { - return from(null, value, encodingOrOffset, length) -}; + list = events[type]; + if (!list) + return this; -if (Buffer.TYPED_ARRAY_SUPPORT) { - Buffer.prototype.__proto__ = Uint8Array.prototype; - Buffer.__proto__ = Uint8Array; -} + if (list === listener || (list.listener && list.listener === listener)) { + if (--this._eventsCount === 0) + this._events = new EventHandlers(); + else { + delete events[type]; + if (events.removeListener) + this.emit('removeListener', type, list.listener || listener); + } + } else if (typeof list !== 'function') { + position = -1; -function assertSize (size) { - if (typeof size !== 'number') { - throw new TypeError('"size" argument must be a number') - } else if (size < 0) { - throw new RangeError('"size" argument must not be negative') - } -} + for (i = list.length; i-- > 0;) { + if (list[i] === listener || + (list[i].listener && list[i].listener === listener)) { + originalListener = list[i].listener; + position = i; + break; + } + } -function alloc (that, size, fill, encoding) { - assertSize(size); - if (size <= 0) { - return createBuffer(that, size) - } - if (fill !== undefined) { - // Only pay attention to encoding if it's a string. This - // prevents accidentally sending in a number that would - // be interpretted as a start offset. - return typeof encoding === 'string' - ? createBuffer(that, size).fill(fill, encoding) - : createBuffer(that, size).fill(fill) - } - return createBuffer(that, size) -} + if (position < 0) + return this; -/** - * Creates a new filled Buffer instance. - * alloc(size[, fill[, encoding]]) - **/ -Buffer.alloc = function (size, fill, encoding) { - return alloc(null, size, fill, encoding) -}; + if (list.length === 1) { + list[0] = undefined; + if (--this._eventsCount === 0) { + this._events = new EventHandlers(); + return this; + } else { + delete events[type]; + } + } else { + spliceOne(list, position); + } -function allocUnsafe (that, size) { - assertSize(size); - that = createBuffer(that, size < 0 ? 0 : checked(size) | 0); - if (!Buffer.TYPED_ARRAY_SUPPORT) { - for (var i = 0; i < size; ++i) { - that[i] = 0; - } - } - return that -} + if (events.removeListener) + this.emit('removeListener', type, originalListener || listener); + } -/** - * Equivalent to Buffer(num), by default creates a non-zero-filled Buffer instance. - * */ -Buffer.allocUnsafe = function (size) { - return allocUnsafe(null, size) -}; -/** - * Equivalent to SlowBuffer(num), by default creates a non-zero-filled Buffer instance. - */ -Buffer.allocUnsafeSlow = function (size) { - return allocUnsafe(null, size) -}; + return this; + }; -function fromString (that, string, encoding) { - if (typeof encoding !== 'string' || encoding === '') { - encoding = 'utf8'; - } +EventEmitter.prototype.removeAllListeners = + function removeAllListeners(type) { + var listeners, events; - if (!Buffer.isEncoding(encoding)) { - throw new TypeError('"encoding" must be a valid string encoding') - } + events = this._events; + if (!events) + return this; - var length = byteLength(string, encoding) | 0; - that = createBuffer(that, length); + // not listening for removeListener, no need to emit + if (!events.removeListener) { + if (arguments.length === 0) { + this._events = new EventHandlers(); + this._eventsCount = 0; + } else if (events[type]) { + if (--this._eventsCount === 0) + this._events = new EventHandlers(); + else + delete events[type]; + } + return this; + } - var actual = that.write(string, encoding); + // emit removeListener for all listeners on all events + if (arguments.length === 0) { + var keys = Object.keys(events); + for (var i = 0, key; i < keys.length; ++i) { + key = keys[i]; + if (key === 'removeListener') continue; + this.removeAllListeners(key); + } + this.removeAllListeners('removeListener'); + this._events = new EventHandlers(); + this._eventsCount = 0; + return this; + } - if (actual !== length) { - // Writing a hex string, for example, that contains invalid characters will - // cause everything after the first invalid character to be ignored. (e.g. - // 'abxxcd' will be treated as 'ab') - that = that.slice(0, actual); - } + listeners = events[type]; - return that -} + if (typeof listeners === 'function') { + this.removeListener(type, listeners); + } else if (listeners) { + // LIFO order + do { + this.removeListener(type, listeners[listeners.length - 1]); + } while (listeners[0]); + } -function fromArrayLike (that, array) { - var length = array.length < 0 ? 0 : checked(array.length) | 0; - that = createBuffer(that, length); - for (var i = 0; i < length; i += 1) { - that[i] = array[i] & 255; - } - return that -} + return this; + }; -function fromArrayBuffer (that, array, byteOffset, length) { - array.byteLength; // this throws if `array` is not a valid ArrayBuffer +EventEmitter.prototype.listeners = function listeners(type) { + var evlistener; + var ret; + var events = this._events; - if (byteOffset < 0 || array.byteLength < byteOffset) { - throw new RangeError('\'offset\' is out of bounds') + if (!events) + ret = []; + else { + evlistener = events[type]; + if (!evlistener) + ret = []; + else if (typeof evlistener === 'function') + ret = [evlistener.listener || evlistener]; + else + ret = unwrapListeners(evlistener); } - if (array.byteLength < byteOffset + (length || 0)) { - throw new RangeError('\'length\' is out of bounds') - } + return ret; +}; - if (byteOffset === undefined && length === undefined) { - array = new Uint8Array(array); - } else if (length === undefined) { - array = new Uint8Array(array, byteOffset); +EventEmitter.listenerCount = function(emitter, type) { + if (typeof emitter.listenerCount === 'function') { + return emitter.listenerCount(type); } else { - array = new Uint8Array(array, byteOffset, length); + return listenerCount$1.call(emitter, type); } +}; - if (Buffer.TYPED_ARRAY_SUPPORT) { - // Return an augmented `Uint8Array` instance, for best performance - that = array; - that.__proto__ = Buffer.prototype; - } else { - // Fallback: Return an object instance of the Buffer class - that = fromArrayLike(that, array); - } - return that -} +EventEmitter.prototype.listenerCount = listenerCount$1; +function listenerCount$1(type) { + var events = this._events; -function fromObject (that, obj) { - if (internalIsBuffer(obj)) { - var len = checked(obj.length) | 0; - that = createBuffer(that, len); + if (events) { + var evlistener = events[type]; - if (that.length === 0) { - return that + if (typeof evlistener === 'function') { + return 1; + } else if (evlistener) { + return evlistener.length; } - - obj.copy(that, 0, 0, len); - return that } - if (obj) { - if ((typeof ArrayBuffer !== 'undefined' && - obj.buffer instanceof ArrayBuffer) || 'length' in obj) { - if (typeof obj.length !== 'number' || isnan(obj.length)) { - return createBuffer(that, 0) - } - return fromArrayLike(that, obj) - } + return 0; +} - if (obj.type === 'Buffer' && isArray$1(obj.data)) { - return fromArrayLike(that, obj.data) - } - } +EventEmitter.prototype.eventNames = function eventNames() { + return this._eventsCount > 0 ? Reflect.ownKeys(this._events) : []; +}; - throw new TypeError('First argument must be a string, Buffer, ArrayBuffer, Array, or array-like object.') +// About 1.5x faster than the two-arg version of Array#splice(). +function spliceOne(list, index) { + for (var i = index, k = i + 1, n = list.length; k < n; i += 1, k += 1) + list[i] = list[k]; + list.pop(); } -function checked (length) { - // Note: cannot use `length < kMaxLength()` here because that fails when - // length is NaN (which is otherwise coerced to zero.) - if (length >= kMaxLength()) { - throw new RangeError('Attempt to allocate Buffer larger than maximum ' + - 'size: 0x' + kMaxLength().toString(16) + ' bytes') - } - return length | 0 -} -Buffer.isBuffer = isBuffer; -function internalIsBuffer (b) { - return !!(b != null && b._isBuffer) +function arrayClone(arr, i) { + var copy = new Array(i); + while (i--) + copy[i] = arr[i]; + return copy; } -Buffer.compare = function compare (a, b) { - if (!internalIsBuffer(a) || !internalIsBuffer(b)) { - throw new TypeError('Arguments must be Buffers') +function unwrapListeners(arr) { + var ret = new Array(arr.length); + for (var i = 0; i < ret.length; ++i) { + ret[i] = arr[i].listener || arr[i]; } + return ret; +} - if (a === b) return 0 - - var x = a.length; - var y = b.length; +var global$1 = (typeof global !== "undefined" ? global : + typeof self !== "undefined" ? self : + typeof window !== "undefined" ? window : {}); - for (var i = 0, len = Math.min(x, y); i < len; ++i) { - if (a[i] !== b[i]) { - x = a[i]; - y = b[i]; - break - } +var lookup = []; +var revLookup = []; +var Arr = typeof Uint8Array !== 'undefined' ? Uint8Array : Array; +var inited = false; +function init () { + inited = true; + var code = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'; + for (var i = 0, len = code.length; i < len; ++i) { + lookup[i] = code[i]; + revLookup[code.charCodeAt(i)] = i; } - if (x < y) return -1 - if (y < x) return 1 - return 0 -}; + revLookup['-'.charCodeAt(0)] = 62; + revLookup['_'.charCodeAt(0)] = 63; +} -Buffer.isEncoding = function isEncoding (encoding) { - switch (String(encoding).toLowerCase()) { - case 'hex': - case 'utf8': - case 'utf-8': - case 'ascii': - case 'latin1': - case 'binary': - case 'base64': - case 'ucs2': - case 'ucs-2': - case 'utf16le': - case 'utf-16le': - return true - default: - return false +function toByteArray (b64) { + if (!inited) { + init(); } -}; + var i, j, l, tmp, placeHolders, arr; + var len = b64.length; -Buffer.concat = function concat (list, length) { - if (!isArray$1(list)) { - throw new TypeError('"list" argument must be an Array of Buffers') + if (len % 4 > 0) { + throw new Error('Invalid string. Length must be a multiple of 4') } - if (list.length === 0) { - return Buffer.alloc(0) - } + // the number of equal signs (place holders) + // if there are two placeholders, than the two characters before it + // represent one byte + // if there is only one, then the three characters before it represent 2 bytes + // this is just a cheap hack to not do indexOf twice + placeHolders = b64[len - 2] === '=' ? 2 : b64[len - 1] === '=' ? 1 : 0; - var i; - if (length === undefined) { - length = 0; - for (i = 0; i < list.length; ++i) { - length += list[i].length; - } - } + // base64 is 4/3 + up to two characters of the original data + arr = new Arr(len * 3 / 4 - placeHolders); - var buffer = Buffer.allocUnsafe(length); - var pos = 0; - for (i = 0; i < list.length; ++i) { - var buf = list[i]; - if (!internalIsBuffer(buf)) { - throw new TypeError('"list" argument must be an Array of Buffers') - } - buf.copy(buffer, pos); - pos += buf.length; - } - return buffer -}; + // if there are placeholders, only get up to the last complete 4 chars + l = placeHolders > 0 ? len - 4 : len; -function byteLength (string, encoding) { - if (internalIsBuffer(string)) { - return string.length - } - if (typeof ArrayBuffer !== 'undefined' && typeof ArrayBuffer.isView === 'function' && - (ArrayBuffer.isView(string) || string instanceof ArrayBuffer)) { - return string.byteLength - } - if (typeof string !== 'string') { - string = '' + string; - } + var L = 0; - var len = string.length; - if (len === 0) return 0 + for (i = 0, j = 0; i < l; i += 4, j += 3) { + tmp = (revLookup[b64.charCodeAt(i)] << 18) | (revLookup[b64.charCodeAt(i + 1)] << 12) | (revLookup[b64.charCodeAt(i + 2)] << 6) | revLookup[b64.charCodeAt(i + 3)]; + arr[L++] = (tmp >> 16) & 0xFF; + arr[L++] = (tmp >> 8) & 0xFF; + arr[L++] = tmp & 0xFF; + } - // Use a for loop to avoid recursion - var loweredCase = false; - for (;;) { - switch (encoding) { - case 'ascii': - case 'latin1': - case 'binary': - return len - case 'utf8': - case 'utf-8': - case undefined: - return utf8ToBytes(string).length - case 'ucs2': - case 'ucs-2': - case 'utf16le': - case 'utf-16le': - return len * 2 - case 'hex': - return len >>> 1 - case 'base64': - return base64ToBytes(string).length - default: - if (loweredCase) return utf8ToBytes(string).length // assume utf8 - encoding = ('' + encoding).toLowerCase(); - loweredCase = true; - } + if (placeHolders === 2) { + tmp = (revLookup[b64.charCodeAt(i)] << 2) | (revLookup[b64.charCodeAt(i + 1)] >> 4); + arr[L++] = tmp & 0xFF; + } else if (placeHolders === 1) { + tmp = (revLookup[b64.charCodeAt(i)] << 10) | (revLookup[b64.charCodeAt(i + 1)] << 4) | (revLookup[b64.charCodeAt(i + 2)] >> 2); + arr[L++] = (tmp >> 8) & 0xFF; + arr[L++] = tmp & 0xFF; } -} -Buffer.byteLength = byteLength; -function slowToString (encoding, start, end) { - var loweredCase = false; + return arr +} - // No need to verify that "this.length <= MAX_UINT32" since it's a read-only - // property of a typed array. +function tripletToBase64 (num) { + return lookup[num >> 18 & 0x3F] + lookup[num >> 12 & 0x3F] + lookup[num >> 6 & 0x3F] + lookup[num & 0x3F] +} - // This behaves neither like String nor Uint8Array in that we set start/end - // to their upper/lower bounds if the value passed is out of range. - // undefined is handled specially as per ECMA-262 6th Edition, - // Section 13.3.3.7 Runtime Semantics: KeyedBindingInitialization. - if (start === undefined || start < 0) { - start = 0; - } - // Return early if start > this.length. Done here to prevent potential uint32 - // coercion fail below. - if (start > this.length) { - return '' +function encodeChunk (uint8, start, end) { + var tmp; + var output = []; + for (var i = start; i < end; i += 3) { + tmp = (uint8[i] << 16) + (uint8[i + 1] << 8) + (uint8[i + 2]); + output.push(tripletToBase64(tmp)); } + return output.join('') +} - if (end === undefined || end > this.length) { - end = this.length; +function fromByteArray (uint8) { + if (!inited) { + init(); } + var tmp; + var len = uint8.length; + var extraBytes = len % 3; // if we have 1 byte left, pad 2 bytes + var output = ''; + var parts = []; + var maxChunkLength = 16383; // must be multiple of 3 - if (end <= 0) { - return '' + // go through the array every three bytes, we'll deal with trailing stuff later + for (var i = 0, len2 = len - extraBytes; i < len2; i += maxChunkLength) { + parts.push(encodeChunk(uint8, i, (i + maxChunkLength) > len2 ? len2 : (i + maxChunkLength))); } - // Force coersion to uint32. This will also coerce falsey/NaN values to 0. - end >>>= 0; - start >>>= 0; - - if (end <= start) { - return '' + // pad the end with zeros, but make sure to not forget the extra bytes + if (extraBytes === 1) { + tmp = uint8[len - 1]; + output += lookup[tmp >> 2]; + output += lookup[(tmp << 4) & 0x3F]; + output += '=='; + } else if (extraBytes === 2) { + tmp = (uint8[len - 2] << 8) + (uint8[len - 1]); + output += lookup[tmp >> 10]; + output += lookup[(tmp >> 4) & 0x3F]; + output += lookup[(tmp << 2) & 0x3F]; + output += '='; } - if (!encoding) encoding = 'utf8'; + parts.push(output); - while (true) { - switch (encoding) { - case 'hex': - return hexSlice(this, start, end) + return parts.join('') +} - case 'utf8': - case 'utf-8': - return utf8Slice(this, start, end) +function read (buffer, offset, isLE, mLen, nBytes) { + var e, m; + var eLen = nBytes * 8 - mLen - 1; + var eMax = (1 << eLen) - 1; + var eBias = eMax >> 1; + var nBits = -7; + var i = isLE ? (nBytes - 1) : 0; + var d = isLE ? -1 : 1; + var s = buffer[offset + i]; - case 'ascii': - return asciiSlice(this, start, end) + i += d; - case 'latin1': - case 'binary': - return latin1Slice(this, start, end) + e = s & ((1 << (-nBits)) - 1); + s >>= (-nBits); + nBits += eLen; + for (; nBits > 0; e = e * 256 + buffer[offset + i], i += d, nBits -= 8) {} - case 'base64': - return base64Slice(this, start, end) + m = e & ((1 << (-nBits)) - 1); + e >>= (-nBits); + nBits += mLen; + for (; nBits > 0; m = m * 256 + buffer[offset + i], i += d, nBits -= 8) {} - case 'ucs2': - case 'ucs-2': - case 'utf16le': - case 'utf-16le': - return utf16leSlice(this, start, end) - - default: - if (loweredCase) throw new TypeError('Unknown encoding: ' + encoding) - encoding = (encoding + '').toLowerCase(); - loweredCase = true; - } + if (e === 0) { + e = 1 - eBias; + } else if (e === eMax) { + return m ? NaN : ((s ? -1 : 1) * Infinity) + } else { + m = m + Math.pow(2, mLen); + e = e - eBias; } + return (s ? -1 : 1) * m * Math.pow(2, e - mLen) } -// The property is used by `Buffer.isBuffer` and `is-buffer` (in Safari 5-7) to detect -// Buffer instances. -Buffer.prototype._isBuffer = true; +function write (buffer, value, offset, isLE, mLen, nBytes) { + var e, m, c; + var eLen = nBytes * 8 - mLen - 1; + var eMax = (1 << eLen) - 1; + var eBias = eMax >> 1; + var rt = (mLen === 23 ? Math.pow(2, -24) - Math.pow(2, -77) : 0); + var i = isLE ? 0 : (nBytes - 1); + var d = isLE ? 1 : -1; + var s = value < 0 || (value === 0 && 1 / value < 0) ? 1 : 0; -function swap (b, n, m) { - var i = b[n]; - b[n] = b[m]; - b[m] = i; -} + value = Math.abs(value); -Buffer.prototype.swap16 = function swap16 () { - var len = this.length; - if (len % 2 !== 0) { - throw new RangeError('Buffer size must be a multiple of 16-bits') - } - for (var i = 0; i < len; i += 2) { - swap(this, i, i + 1); - } - return this -}; + if (isNaN(value) || value === Infinity) { + m = isNaN(value) ? 1 : 0; + e = eMax; + } else { + e = Math.floor(Math.log(value) / Math.LN2); + if (value * (c = Math.pow(2, -e)) < 1) { + e--; + c *= 2; + } + if (e + eBias >= 1) { + value += rt / c; + } else { + value += rt * Math.pow(2, 1 - eBias); + } + if (value * c >= 2) { + e++; + c /= 2; + } -Buffer.prototype.swap32 = function swap32 () { - var len = this.length; - if (len % 4 !== 0) { - throw new RangeError('Buffer size must be a multiple of 32-bits') - } - for (var i = 0; i < len; i += 4) { - swap(this, i, i + 3); - swap(this, i + 1, i + 2); + if (e + eBias >= eMax) { + m = 0; + e = eMax; + } else if (e + eBias >= 1) { + m = (value * c - 1) * Math.pow(2, mLen); + e = e + eBias; + } else { + m = value * Math.pow(2, eBias - 1) * Math.pow(2, mLen); + e = 0; + } } - return this -}; -Buffer.prototype.swap64 = function swap64 () { - var len = this.length; - if (len % 8 !== 0) { - throw new RangeError('Buffer size must be a multiple of 64-bits') - } - for (var i = 0; i < len; i += 8) { - swap(this, i, i + 7); - swap(this, i + 1, i + 6); - swap(this, i + 2, i + 5); - swap(this, i + 3, i + 4); - } - return this -}; + for (; mLen >= 8; buffer[offset + i] = m & 0xff, i += d, m /= 256, mLen -= 8) {} -Buffer.prototype.toString = function toString () { - var length = this.length | 0; - if (length === 0) return '' - if (arguments.length === 0) return utf8Slice(this, 0, length) - return slowToString.apply(this, arguments) -}; + e = (e << mLen) | m; + eLen += mLen; + for (; eLen > 0; buffer[offset + i] = e & 0xff, i += d, e /= 256, eLen -= 8) {} -Buffer.prototype.equals = function equals (b) { - if (!internalIsBuffer(b)) throw new TypeError('Argument must be a Buffer') - if (this === b) return true - return Buffer.compare(this, b) === 0 -}; + buffer[offset + i - d] |= s * 128; +} -Buffer.prototype.inspect = function inspect () { - var str = ''; - var max = INSPECT_MAX_BYTES; - if (this.length > 0) { - str = this.toString('hex', 0, max).match(/.{2}/g).join(' '); - if (this.length > max) str += ' ... '; - } - return '' +var toString = {}.toString; + +var isArray$1 = Array.isArray || function (arr) { + return toString.call(arr) == '[object Array]'; }; -Buffer.prototype.compare = function compare (target, start, end, thisStart, thisEnd) { - if (!internalIsBuffer(target)) { - throw new TypeError('Argument must be a Buffer') - } +var INSPECT_MAX_BYTES = 50; - if (start === undefined) { - start = 0; - } - if (end === undefined) { - end = target ? target.length : 0; - } - if (thisStart === undefined) { - thisStart = 0; - } - if (thisEnd === undefined) { - thisEnd = this.length; - } +/** + * If `Buffer.TYPED_ARRAY_SUPPORT`: + * === true Use Uint8Array implementation (fastest) + * === false Use Object implementation (most compatible, even IE6) + * + * Browsers that support typed arrays are IE 10+, Firefox 4+, Chrome 7+, Safari 5.1+, + * Opera 11.6+, iOS 4.2+. + * + * Due to various browser bugs, sometimes the Object implementation will be used even + * when the browser supports typed arrays. + * + * Note: + * + * - Firefox 4-29 lacks support for adding new properties to `Uint8Array` instances, + * See: https://bugzilla.mozilla.org/show_bug.cgi?id=695438. + * + * - Chrome 9-10 is missing the `TypedArray.prototype.subarray` function. + * + * - IE10 has a broken `TypedArray.prototype.subarray` function which returns arrays of + * incorrect length in some situations. - if (start < 0 || end > target.length || thisStart < 0 || thisEnd > this.length) { - throw new RangeError('out of range index') - } + * We detect these buggy browsers and set `Buffer.TYPED_ARRAY_SUPPORT` to `false` so they + * get the Object implementation, which is slower but behaves correctly. + */ +Buffer.TYPED_ARRAY_SUPPORT = global$1.TYPED_ARRAY_SUPPORT !== undefined + ? global$1.TYPED_ARRAY_SUPPORT + : true; - if (thisStart >= thisEnd && start >= end) { - return 0 - } - if (thisStart >= thisEnd) { - return -1 +function kMaxLength () { + return Buffer.TYPED_ARRAY_SUPPORT + ? 0x7fffffff + : 0x3fffffff +} + +function createBuffer (that, length) { + if (kMaxLength() < length) { + throw new RangeError('Invalid typed array length') } - if (start >= end) { - return 1 + if (Buffer.TYPED_ARRAY_SUPPORT) { + // Return an augmented `Uint8Array` instance, for best performance + that = new Uint8Array(length); + that.__proto__ = Buffer.prototype; + } else { + // Fallback: Return an object instance of the Buffer class + if (that === null) { + that = new Buffer(length); + } + that.length = length; } - start >>>= 0; - end >>>= 0; - thisStart >>>= 0; - thisEnd >>>= 0; - - if (this === target) return 0 + return that +} - var x = thisEnd - thisStart; - var y = end - start; - var len = Math.min(x, y); +/** + * The Buffer constructor returns instances of `Uint8Array` that have their + * prototype changed to `Buffer.prototype`. Furthermore, `Buffer` is a subclass of + * `Uint8Array`, so the returned instances will have all the node `Buffer` methods + * and the `Uint8Array` methods. Square bracket notation works as expected -- it + * returns a single octet. + * + * The `Uint8Array` prototype remains unmodified. + */ - var thisCopy = this.slice(thisStart, thisEnd); - var targetCopy = target.slice(start, end); +function Buffer (arg, encodingOrOffset, length) { + if (!Buffer.TYPED_ARRAY_SUPPORT && !(this instanceof Buffer)) { + return new Buffer(arg, encodingOrOffset, length) + } - for (var i = 0; i < len; ++i) { - if (thisCopy[i] !== targetCopy[i]) { - x = thisCopy[i]; - y = targetCopy[i]; - break + // Common case. + if (typeof arg === 'number') { + if (typeof encodingOrOffset === 'string') { + throw new Error( + 'If encoding is specified then the first argument must be a string' + ) } + return allocUnsafe(this, arg) } + return from(this, arg, encodingOrOffset, length) +} - if (x < y) return -1 - if (y < x) return 1 - return 0 +Buffer.poolSize = 8192; // not used by this implementation + +// TODO: Legacy, not needed anymore. Remove in next major version. +Buffer._augment = function (arr) { + arr.__proto__ = Buffer.prototype; + return arr }; -// Finds either the first index of `val` in `buffer` at offset >= `byteOffset`, -// OR the last index of `val` in `buffer` at offset <= `byteOffset`. -// -// Arguments: -// - buffer - a Buffer to search -// - val - a string, Buffer, or number -// - byteOffset - an index into `buffer`; will be clamped to an int32 -// - encoding - an optional encoding, relevant is val is a string -// - dir - true for indexOf, false for lastIndexOf -function bidirectionalIndexOf (buffer, val, byteOffset, encoding, dir) { - // Empty buffer means no match - if (buffer.length === 0) return -1 - - // Normalize byteOffset - if (typeof byteOffset === 'string') { - encoding = byteOffset; - byteOffset = 0; - } else if (byteOffset > 0x7fffffff) { - byteOffset = 0x7fffffff; - } else if (byteOffset < -0x80000000) { - byteOffset = -0x80000000; - } - byteOffset = +byteOffset; // Coerce to Number. - if (isNaN(byteOffset)) { - // byteOffset: it it's undefined, null, NaN, "foo", etc, search whole buffer - byteOffset = dir ? 0 : (buffer.length - 1); - } - - // Normalize byteOffset: negative offsets start from the end of the buffer - if (byteOffset < 0) byteOffset = buffer.length + byteOffset; - if (byteOffset >= buffer.length) { - if (dir) return -1 - else byteOffset = buffer.length - 1; - } else if (byteOffset < 0) { - if (dir) byteOffset = 0; - else return -1 +function from (that, value, encodingOrOffset, length) { + if (typeof value === 'number') { + throw new TypeError('"value" argument must not be a number') } - // Normalize val - if (typeof val === 'string') { - val = Buffer.from(val, encoding); + if (typeof ArrayBuffer !== 'undefined' && value instanceof ArrayBuffer) { + return fromArrayBuffer(that, value, encodingOrOffset, length) } - // Finally, search either indexOf (if dir is true) or lastIndexOf - if (internalIsBuffer(val)) { - // Special case: looking for empty string/buffer always fails - if (val.length === 0) { - return -1 - } - return arrayIndexOf(buffer, val, byteOffset, encoding, dir) - } else if (typeof val === 'number') { - val = val & 0xFF; // Search for a byte value [0-255] - if (Buffer.TYPED_ARRAY_SUPPORT && - typeof Uint8Array.prototype.indexOf === 'function') { - if (dir) { - return Uint8Array.prototype.indexOf.call(buffer, val, byteOffset) - } else { - return Uint8Array.prototype.lastIndexOf.call(buffer, val, byteOffset) - } - } - return arrayIndexOf(buffer, [ val ], byteOffset, encoding, dir) + if (typeof value === 'string') { + return fromString(that, value, encodingOrOffset) } - throw new TypeError('val must be string, number or Buffer') + return fromObject(that, value) } -function arrayIndexOf (arr, val, byteOffset, encoding, dir) { - var indexSize = 1; - var arrLength = arr.length; - var valLength = val.length; +/** + * Functionally equivalent to Buffer(arg, encoding) but throws a TypeError + * if value is a number. + * Buffer.from(str[, encoding]) + * Buffer.from(array) + * Buffer.from(buffer) + * Buffer.from(arrayBuffer[, byteOffset[, length]]) + **/ +Buffer.from = function (value, encodingOrOffset, length) { + return from(null, value, encodingOrOffset, length) +}; - if (encoding !== undefined) { - encoding = String(encoding).toLowerCase(); - if (encoding === 'ucs2' || encoding === 'ucs-2' || - encoding === 'utf16le' || encoding === 'utf-16le') { - if (arr.length < 2 || val.length < 2) { - return -1 - } - indexSize = 2; - arrLength /= 2; - valLength /= 2; - byteOffset /= 2; - } +if (Buffer.TYPED_ARRAY_SUPPORT) { + Buffer.prototype.__proto__ = Uint8Array.prototype; + Buffer.__proto__ = Uint8Array; +} + +function assertSize (size) { + if (typeof size !== 'number') { + throw new TypeError('"size" argument must be a number') + } else if (size < 0) { + throw new RangeError('"size" argument must not be negative') } +} - function read (buf, i) { - if (indexSize === 1) { - return buf[i] - } else { - return buf.readUInt16BE(i * indexSize) - } +function alloc (that, size, fill, encoding) { + assertSize(size); + if (size <= 0) { + return createBuffer(that, size) + } + if (fill !== undefined) { + // Only pay attention to encoding if it's a string. This + // prevents accidentally sending in a number that would + // be interpretted as a start offset. + return typeof encoding === 'string' + ? createBuffer(that, size).fill(fill, encoding) + : createBuffer(that, size).fill(fill) } + return createBuffer(that, size) +} - var i; - if (dir) { - var foundIndex = -1; - for (i = byteOffset; i < arrLength; i++) { - if (read(arr, i) === read(val, foundIndex === -1 ? 0 : i - foundIndex)) { - if (foundIndex === -1) foundIndex = i; - if (i - foundIndex + 1 === valLength) return foundIndex * indexSize - } else { - if (foundIndex !== -1) i -= i - foundIndex; - foundIndex = -1; - } - } - } else { - if (byteOffset + valLength > arrLength) byteOffset = arrLength - valLength; - for (i = byteOffset; i >= 0; i--) { - var found = true; - for (var j = 0; j < valLength; j++) { - if (read(arr, i + j) !== read(val, j)) { - found = false; - break - } - } - if (found) return i +/** + * Creates a new filled Buffer instance. + * alloc(size[, fill[, encoding]]) + **/ +Buffer.alloc = function (size, fill, encoding) { + return alloc(null, size, fill, encoding) +}; + +function allocUnsafe (that, size) { + assertSize(size); + that = createBuffer(that, size < 0 ? 0 : checked(size) | 0); + if (!Buffer.TYPED_ARRAY_SUPPORT) { + for (var i = 0; i < size; ++i) { + that[i] = 0; } } - - return -1 + return that } -Buffer.prototype.includes = function includes (val, byteOffset, encoding) { - return this.indexOf(val, byteOffset, encoding) !== -1 +/** + * Equivalent to Buffer(num), by default creates a non-zero-filled Buffer instance. + * */ +Buffer.allocUnsafe = function (size) { + return allocUnsafe(null, size) }; - -Buffer.prototype.indexOf = function indexOf (val, byteOffset, encoding) { - return bidirectionalIndexOf(this, val, byteOffset, encoding, true) +/** + * Equivalent to SlowBuffer(num), by default creates a non-zero-filled Buffer instance. + */ +Buffer.allocUnsafeSlow = function (size) { + return allocUnsafe(null, size) }; -Buffer.prototype.lastIndexOf = function lastIndexOf (val, byteOffset, encoding) { - return bidirectionalIndexOf(this, val, byteOffset, encoding, false) -}; +function fromString (that, string, encoding) { + if (typeof encoding !== 'string' || encoding === '') { + encoding = 'utf8'; + } -function hexWrite (buf, string, offset, length) { - offset = Number(offset) || 0; - var remaining = buf.length - offset; - if (!length) { - length = remaining; - } else { - length = Number(length); - if (length > remaining) { - length = remaining; - } + if (!Buffer.isEncoding(encoding)) { + throw new TypeError('"encoding" must be a valid string encoding') } - // must be an even number of digits - var strLen = string.length; - if (strLen % 2 !== 0) throw new TypeError('Invalid hex string') + var length = byteLength(string, encoding) | 0; + that = createBuffer(that, length); - if (length > strLen / 2) { - length = strLen / 2; - } - for (var i = 0; i < length; ++i) { - var parsed = parseInt(string.substr(i * 2, 2), 16); - if (isNaN(parsed)) return i - buf[offset + i] = parsed; + var actual = that.write(string, encoding); + + if (actual !== length) { + // Writing a hex string, for example, that contains invalid characters will + // cause everything after the first invalid character to be ignored. (e.g. + // 'abxxcd' will be treated as 'ab') + that = that.slice(0, actual); } - return i -} -function utf8Write (buf, string, offset, length) { - return blitBuffer(utf8ToBytes(string, buf.length - offset), buf, offset, length) + return that } -function asciiWrite (buf, string, offset, length) { - return blitBuffer(asciiToBytes(string), buf, offset, length) +function fromArrayLike (that, array) { + var length = array.length < 0 ? 0 : checked(array.length) | 0; + that = createBuffer(that, length); + for (var i = 0; i < length; i += 1) { + that[i] = array[i] & 255; + } + return that } -function latin1Write (buf, string, offset, length) { - return asciiWrite(buf, string, offset, length) -} +function fromArrayBuffer (that, array, byteOffset, length) { + array.byteLength; // this throws if `array` is not a valid ArrayBuffer -function base64Write (buf, string, offset, length) { - return blitBuffer(base64ToBytes(string), buf, offset, length) -} + if (byteOffset < 0 || array.byteLength < byteOffset) { + throw new RangeError('\'offset\' is out of bounds') + } -function ucs2Write (buf, string, offset, length) { - return blitBuffer(utf16leToBytes(string, buf.length - offset), buf, offset, length) -} + if (array.byteLength < byteOffset + (length || 0)) { + throw new RangeError('\'length\' is out of bounds') + } -Buffer.prototype.write = function write (string, offset, length, encoding) { - // Buffer#write(string) - if (offset === undefined) { - encoding = 'utf8'; - length = this.length; - offset = 0; - // Buffer#write(string, encoding) - } else if (length === undefined && typeof offset === 'string') { - encoding = offset; - length = this.length; - offset = 0; - // Buffer#write(string, offset[, length][, encoding]) - } else if (isFinite(offset)) { - offset = offset | 0; - if (isFinite(length)) { - length = length | 0; - if (encoding === undefined) encoding = 'utf8'; - } else { - encoding = length; - length = undefined; - } - // legacy write(string, encoding, offset, length) - remove in v0.13 + if (byteOffset === undefined && length === undefined) { + array = new Uint8Array(array); + } else if (length === undefined) { + array = new Uint8Array(array, byteOffset); } else { - throw new Error( - 'Buffer.write(string, encoding, offset[, length]) is no longer supported' - ) + array = new Uint8Array(array, byteOffset, length); } - var remaining = this.length - offset; - if (length === undefined || length > remaining) length = remaining; + if (Buffer.TYPED_ARRAY_SUPPORT) { + // Return an augmented `Uint8Array` instance, for best performance + that = array; + that.__proto__ = Buffer.prototype; + } else { + // Fallback: Return an object instance of the Buffer class + that = fromArrayLike(that, array); + } + return that +} - if ((string.length > 0 && (length < 0 || offset < 0)) || offset > this.length) { - throw new RangeError('Attempt to write outside buffer bounds') +function fromObject (that, obj) { + if (internalIsBuffer(obj)) { + var len = checked(obj.length) | 0; + that = createBuffer(that, len); + + if (that.length === 0) { + return that + } + + obj.copy(that, 0, 0, len); + return that } - if (!encoding) encoding = 'utf8'; + if (obj) { + if ((typeof ArrayBuffer !== 'undefined' && + obj.buffer instanceof ArrayBuffer) || 'length' in obj) { + if (typeof obj.length !== 'number' || isnan(obj.length)) { + return createBuffer(that, 0) + } + return fromArrayLike(that, obj) + } + + if (obj.type === 'Buffer' && isArray$1(obj.data)) { + return fromArrayLike(that, obj.data) + } + } + + throw new TypeError('First argument must be a string, Buffer, ArrayBuffer, Array, or array-like object.') +} + +function checked (length) { + // Note: cannot use `length < kMaxLength()` here because that fails when + // length is NaN (which is otherwise coerced to zero.) + if (length >= kMaxLength()) { + throw new RangeError('Attempt to allocate Buffer larger than maximum ' + + 'size: 0x' + kMaxLength().toString(16) + ' bytes') + } + return length | 0 +} +Buffer.isBuffer = isBuffer; +function internalIsBuffer (b) { + return !!(b != null && b._isBuffer) +} + +Buffer.compare = function compare (a, b) { + if (!internalIsBuffer(a) || !internalIsBuffer(b)) { + throw new TypeError('Arguments must be Buffers') + } + + if (a === b) return 0 + + var x = a.length; + var y = b.length; + + for (var i = 0, len = Math.min(x, y); i < len; ++i) { + if (a[i] !== b[i]) { + x = a[i]; + y = b[i]; + break + } + } + + if (x < y) return -1 + if (y < x) return 1 + return 0 +}; + +Buffer.isEncoding = function isEncoding (encoding) { + switch (String(encoding).toLowerCase()) { + case 'hex': + case 'utf8': + case 'utf-8': + case 'ascii': + case 'latin1': + case 'binary': + case 'base64': + case 'ucs2': + case 'ucs-2': + case 'utf16le': + case 'utf-16le': + return true + default: + return false + } +}; + +Buffer.concat = function concat (list, length) { + if (!isArray$1(list)) { + throw new TypeError('"list" argument must be an Array of Buffers') + } + + if (list.length === 0) { + return Buffer.alloc(0) + } + + var i; + if (length === undefined) { + length = 0; + for (i = 0; i < list.length; ++i) { + length += list[i].length; + } + } + + var buffer = Buffer.allocUnsafe(length); + var pos = 0; + for (i = 0; i < list.length; ++i) { + var buf = list[i]; + if (!internalIsBuffer(buf)) { + throw new TypeError('"list" argument must be an Array of Buffers') + } + buf.copy(buffer, pos); + pos += buf.length; + } + return buffer +}; + +function byteLength (string, encoding) { + if (internalIsBuffer(string)) { + return string.length + } + if (typeof ArrayBuffer !== 'undefined' && typeof ArrayBuffer.isView === 'function' && + (ArrayBuffer.isView(string) || string instanceof ArrayBuffer)) { + return string.byteLength + } + if (typeof string !== 'string') { + string = '' + string; + } + + var len = string.length; + if (len === 0) return 0 + // Use a for loop to avoid recursion var loweredCase = false; for (;;) { switch (encoding) { + case 'ascii': + case 'latin1': + case 'binary': + return len + case 'utf8': + case 'utf-8': + case undefined: + return utf8ToBytes(string).length + case 'ucs2': + case 'ucs-2': + case 'utf16le': + case 'utf-16le': + return len * 2 case 'hex': - return hexWrite(this, string, offset, length) + return len >>> 1 + case 'base64': + return base64ToBytes(string).length + default: + if (loweredCase) return utf8ToBytes(string).length // assume utf8 + encoding = ('' + encoding).toLowerCase(); + loweredCase = true; + } + } +} +Buffer.byteLength = byteLength; + +function slowToString (encoding, start, end) { + var loweredCase = false; + + // No need to verify that "this.length <= MAX_UINT32" since it's a read-only + // property of a typed array. + + // This behaves neither like String nor Uint8Array in that we set start/end + // to their upper/lower bounds if the value passed is out of range. + // undefined is handled specially as per ECMA-262 6th Edition, + // Section 13.3.3.7 Runtime Semantics: KeyedBindingInitialization. + if (start === undefined || start < 0) { + start = 0; + } + // Return early if start > this.length. Done here to prevent potential uint32 + // coercion fail below. + if (start > this.length) { + return '' + } + + if (end === undefined || end > this.length) { + end = this.length; + } + + if (end <= 0) { + return '' + } + + // Force coersion to uint32. This will also coerce falsey/NaN values to 0. + end >>>= 0; + start >>>= 0; + + if (end <= start) { + return '' + } + + if (!encoding) encoding = 'utf8'; + + while (true) { + switch (encoding) { + case 'hex': + return hexSlice(this, start, end) case 'utf8': case 'utf-8': - return utf8Write(this, string, offset, length) + return utf8Slice(this, start, end) case 'ascii': - return asciiWrite(this, string, offset, length) + return asciiSlice(this, start, end) case 'latin1': case 'binary': - return latin1Write(this, string, offset, length) + return latin1Slice(this, start, end) case 'base64': - // Warning: maxLength not taken into account in base64Write - return base64Write(this, string, offset, length) + return base64Slice(this, start, end) case 'ucs2': case 'ucs-2': case 'utf16le': case 'utf-16le': - return ucs2Write(this, string, offset, length) + return utf16leSlice(this, start, end) default: if (loweredCase) throw new TypeError('Unknown encoding: ' + encoding) - encoding = ('' + encoding).toLowerCase(); + encoding = (encoding + '').toLowerCase(); loweredCase = true; } } -}; - -Buffer.prototype.toJSON = function toJSON () { - return { - type: 'Buffer', - data: Array.prototype.slice.call(this._arr || this, 0) - } -}; - -function base64Slice (buf, start, end) { - if (start === 0 && end === buf.length) { - return fromByteArray(buf) - } else { - return fromByteArray(buf.slice(start, end)) - } } -function utf8Slice (buf, start, end) { - end = Math.min(buf.length, end); - var res = []; - - var i = start; - while (i < end) { - var firstByte = buf[i]; - var codePoint = null; - var bytesPerSequence = (firstByte > 0xEF) ? 4 - : (firstByte > 0xDF) ? 3 - : (firstByte > 0xBF) ? 2 - : 1; +// The property is used by `Buffer.isBuffer` and `is-buffer` (in Safari 5-7) to detect +// Buffer instances. +Buffer.prototype._isBuffer = true; - if (i + bytesPerSequence <= end) { - var secondByte, thirdByte, fourthByte, tempCodePoint; +function swap (b, n, m) { + var i = b[n]; + b[n] = b[m]; + b[m] = i; +} - switch (bytesPerSequence) { - case 1: - if (firstByte < 0x80) { - codePoint = firstByte; - } - break - case 2: - secondByte = buf[i + 1]; - if ((secondByte & 0xC0) === 0x80) { - tempCodePoint = (firstByte & 0x1F) << 0x6 | (secondByte & 0x3F); - if (tempCodePoint > 0x7F) { - codePoint = tempCodePoint; - } - } - break - case 3: - secondByte = buf[i + 1]; - thirdByte = buf[i + 2]; - if ((secondByte & 0xC0) === 0x80 && (thirdByte & 0xC0) === 0x80) { - tempCodePoint = (firstByte & 0xF) << 0xC | (secondByte & 0x3F) << 0x6 | (thirdByte & 0x3F); - if (tempCodePoint > 0x7FF && (tempCodePoint < 0xD800 || tempCodePoint > 0xDFFF)) { - codePoint = tempCodePoint; - } - } - break - case 4: - secondByte = buf[i + 1]; - thirdByte = buf[i + 2]; - fourthByte = buf[i + 3]; - if ((secondByte & 0xC0) === 0x80 && (thirdByte & 0xC0) === 0x80 && (fourthByte & 0xC0) === 0x80) { - tempCodePoint = (firstByte & 0xF) << 0x12 | (secondByte & 0x3F) << 0xC | (thirdByte & 0x3F) << 0x6 | (fourthByte & 0x3F); - if (tempCodePoint > 0xFFFF && tempCodePoint < 0x110000) { - codePoint = tempCodePoint; - } - } - } - } +Buffer.prototype.swap16 = function swap16 () { + var len = this.length; + if (len % 2 !== 0) { + throw new RangeError('Buffer size must be a multiple of 16-bits') + } + for (var i = 0; i < len; i += 2) { + swap(this, i, i + 1); + } + return this +}; - if (codePoint === null) { - // we did not generate a valid codePoint so insert a - // replacement char (U+FFFD) and advance only 1 byte - codePoint = 0xFFFD; - bytesPerSequence = 1; - } else if (codePoint > 0xFFFF) { - // encode to utf16 (surrogate pair dance) - codePoint -= 0x10000; - res.push(codePoint >>> 10 & 0x3FF | 0xD800); - codePoint = 0xDC00 | codePoint & 0x3FF; - } +Buffer.prototype.swap32 = function swap32 () { + var len = this.length; + if (len % 4 !== 0) { + throw new RangeError('Buffer size must be a multiple of 32-bits') + } + for (var i = 0; i < len; i += 4) { + swap(this, i, i + 3); + swap(this, i + 1, i + 2); + } + return this +}; - res.push(codePoint); - i += bytesPerSequence; +Buffer.prototype.swap64 = function swap64 () { + var len = this.length; + if (len % 8 !== 0) { + throw new RangeError('Buffer size must be a multiple of 64-bits') + } + for (var i = 0; i < len; i += 8) { + swap(this, i, i + 7); + swap(this, i + 1, i + 6); + swap(this, i + 2, i + 5); + swap(this, i + 3, i + 4); } + return this +}; - return decodeCodePointsArray(res) -} +Buffer.prototype.toString = function toString () { + var length = this.length | 0; + if (length === 0) return '' + if (arguments.length === 0) return utf8Slice(this, 0, length) + return slowToString.apply(this, arguments) +}; -// Based on http://stackoverflow.com/a/22747272/680742, the browser with -// the lowest limit is Chrome, with 0x10000 args. -// We go 1 magnitude less, for safety -var MAX_ARGUMENTS_LENGTH = 0x1000; +Buffer.prototype.equals = function equals (b) { + if (!internalIsBuffer(b)) throw new TypeError('Argument must be a Buffer') + if (this === b) return true + return Buffer.compare(this, b) === 0 +}; -function decodeCodePointsArray (codePoints) { - var len = codePoints.length; - if (len <= MAX_ARGUMENTS_LENGTH) { - return String.fromCharCode.apply(String, codePoints) // avoid extra slice() +Buffer.prototype.inspect = function inspect () { + var str = ''; + var max = INSPECT_MAX_BYTES; + if (this.length > 0) { + str = this.toString('hex', 0, max).match(/.{2}/g).join(' '); + if (this.length > max) str += ' ... '; } + return '' +}; - // Decode in chunks to avoid "call stack size exceeded". - var res = ''; - var i = 0; - while (i < len) { - res += String.fromCharCode.apply( - String, - codePoints.slice(i, i += MAX_ARGUMENTS_LENGTH) - ); +Buffer.prototype.compare = function compare (target, start, end, thisStart, thisEnd) { + if (!internalIsBuffer(target)) { + throw new TypeError('Argument must be a Buffer') } - return res -} -function asciiSlice (buf, start, end) { - var ret = ''; - end = Math.min(buf.length, end); - - for (var i = start; i < end; ++i) { - ret += String.fromCharCode(buf[i] & 0x7F); + if (start === undefined) { + start = 0; } - return ret -} - -function latin1Slice (buf, start, end) { - var ret = ''; - end = Math.min(buf.length, end); - - for (var i = start; i < end; ++i) { - ret += String.fromCharCode(buf[i]); + if (end === undefined) { + end = target ? target.length : 0; + } + if (thisStart === undefined) { + thisStart = 0; + } + if (thisEnd === undefined) { + thisEnd = this.length; } - return ret -} - -function hexSlice (buf, start, end) { - var len = buf.length; - - if (!start || start < 0) start = 0; - if (!end || end < 0 || end > len) end = len; - var out = ''; - for (var i = start; i < end; ++i) { - out += toHex(buf[i]); + if (start < 0 || end > target.length || thisStart < 0 || thisEnd > this.length) { + throw new RangeError('out of range index') } - return out -} -function utf16leSlice (buf, start, end) { - var bytes = buf.slice(start, end); - var res = ''; - for (var i = 0; i < bytes.length; i += 2) { - res += String.fromCharCode(bytes[i] + bytes[i + 1] * 256); + if (thisStart >= thisEnd && start >= end) { + return 0 + } + if (thisStart >= thisEnd) { + return -1 + } + if (start >= end) { + return 1 } - return res -} -Buffer.prototype.slice = function slice (start, end) { - var len = this.length; - start = ~~start; - end = end === undefined ? len : ~~end; + start >>>= 0; + end >>>= 0; + thisStart >>>= 0; + thisEnd >>>= 0; - if (start < 0) { - start += len; - if (start < 0) start = 0; - } else if (start > len) { - start = len; - } + if (this === target) return 0 - if (end < 0) { - end += len; - if (end < 0) end = 0; - } else if (end > len) { - end = len; - } + var x = thisEnd - thisStart; + var y = end - start; + var len = Math.min(x, y); - if (end < start) end = start; + var thisCopy = this.slice(thisStart, thisEnd); + var targetCopy = target.slice(start, end); - var newBuf; - if (Buffer.TYPED_ARRAY_SUPPORT) { - newBuf = this.subarray(start, end); - newBuf.__proto__ = Buffer.prototype; - } else { - var sliceLen = end - start; - newBuf = new Buffer(sliceLen, undefined); - for (var i = 0; i < sliceLen; ++i) { - newBuf[i] = this[i + start]; + for (var i = 0; i < len; ++i) { + if (thisCopy[i] !== targetCopy[i]) { + x = thisCopy[i]; + y = targetCopy[i]; + break } } - return newBuf + if (x < y) return -1 + if (y < x) return 1 + return 0 }; -/* - * Need to make sure that buffer isn't trying to write out of bounds. - */ -function checkOffset (offset, ext, length) { - if ((offset % 1) !== 0 || offset < 0) throw new RangeError('offset is not uint') - if (offset + ext > length) throw new RangeError('Trying to access beyond buffer length') -} - -Buffer.prototype.readUIntLE = function readUIntLE (offset, byteLength, noAssert) { - offset = offset | 0; - byteLength = byteLength | 0; - if (!noAssert) checkOffset(offset, byteLength, this.length); +// Finds either the first index of `val` in `buffer` at offset >= `byteOffset`, +// OR the last index of `val` in `buffer` at offset <= `byteOffset`. +// +// Arguments: +// - buffer - a Buffer to search +// - val - a string, Buffer, or number +// - byteOffset - an index into `buffer`; will be clamped to an int32 +// - encoding - an optional encoding, relevant is val is a string +// - dir - true for indexOf, false for lastIndexOf +function bidirectionalIndexOf (buffer, val, byteOffset, encoding, dir) { + // Empty buffer means no match + if (buffer.length === 0) return -1 - var val = this[offset]; - var mul = 1; - var i = 0; - while (++i < byteLength && (mul *= 0x100)) { - val += this[offset + i] * mul; + // Normalize byteOffset + if (typeof byteOffset === 'string') { + encoding = byteOffset; + byteOffset = 0; + } else if (byteOffset > 0x7fffffff) { + byteOffset = 0x7fffffff; + } else if (byteOffset < -0x80000000) { + byteOffset = -0x80000000; } - - return val -}; - -Buffer.prototype.readUIntBE = function readUIntBE (offset, byteLength, noAssert) { - offset = offset | 0; - byteLength = byteLength | 0; - if (!noAssert) { - checkOffset(offset, byteLength, this.length); + byteOffset = +byteOffset; // Coerce to Number. + if (isNaN(byteOffset)) { + // byteOffset: it it's undefined, null, NaN, "foo", etc, search whole buffer + byteOffset = dir ? 0 : (buffer.length - 1); } - var val = this[offset + --byteLength]; - var mul = 1; - while (byteLength > 0 && (mul *= 0x100)) { - val += this[offset + --byteLength] * mul; + // Normalize byteOffset: negative offsets start from the end of the buffer + if (byteOffset < 0) byteOffset = buffer.length + byteOffset; + if (byteOffset >= buffer.length) { + if (dir) return -1 + else byteOffset = buffer.length - 1; + } else if (byteOffset < 0) { + if (dir) byteOffset = 0; + else return -1 } - return val -}; - -Buffer.prototype.readUInt8 = function readUInt8 (offset, noAssert) { - if (!noAssert) checkOffset(offset, 1, this.length); - return this[offset] -}; - -Buffer.prototype.readUInt16LE = function readUInt16LE (offset, noAssert) { - if (!noAssert) checkOffset(offset, 2, this.length); - return this[offset] | (this[offset + 1] << 8) -}; - -Buffer.prototype.readUInt16BE = function readUInt16BE (offset, noAssert) { - if (!noAssert) checkOffset(offset, 2, this.length); - return (this[offset] << 8) | this[offset + 1] -}; - -Buffer.prototype.readUInt32LE = function readUInt32LE (offset, noAssert) { - if (!noAssert) checkOffset(offset, 4, this.length); - - return ((this[offset]) | - (this[offset + 1] << 8) | - (this[offset + 2] << 16)) + - (this[offset + 3] * 0x1000000) -}; + // Normalize val + if (typeof val === 'string') { + val = Buffer.from(val, encoding); + } -Buffer.prototype.readUInt32BE = function readUInt32BE (offset, noAssert) { - if (!noAssert) checkOffset(offset, 4, this.length); + // Finally, search either indexOf (if dir is true) or lastIndexOf + if (internalIsBuffer(val)) { + // Special case: looking for empty string/buffer always fails + if (val.length === 0) { + return -1 + } + return arrayIndexOf(buffer, val, byteOffset, encoding, dir) + } else if (typeof val === 'number') { + val = val & 0xFF; // Search for a byte value [0-255] + if (Buffer.TYPED_ARRAY_SUPPORT && + typeof Uint8Array.prototype.indexOf === 'function') { + if (dir) { + return Uint8Array.prototype.indexOf.call(buffer, val, byteOffset) + } else { + return Uint8Array.prototype.lastIndexOf.call(buffer, val, byteOffset) + } + } + return arrayIndexOf(buffer, [ val ], byteOffset, encoding, dir) + } - return (this[offset] * 0x1000000) + - ((this[offset + 1] << 16) | - (this[offset + 2] << 8) | - this[offset + 3]) -}; + throw new TypeError('val must be string, number or Buffer') +} -Buffer.prototype.readIntLE = function readIntLE (offset, byteLength, noAssert) { - offset = offset | 0; - byteLength = byteLength | 0; - if (!noAssert) checkOffset(offset, byteLength, this.length); +function arrayIndexOf (arr, val, byteOffset, encoding, dir) { + var indexSize = 1; + var arrLength = arr.length; + var valLength = val.length; - var val = this[offset]; - var mul = 1; - var i = 0; - while (++i < byteLength && (mul *= 0x100)) { - val += this[offset + i] * mul; + if (encoding !== undefined) { + encoding = String(encoding).toLowerCase(); + if (encoding === 'ucs2' || encoding === 'ucs-2' || + encoding === 'utf16le' || encoding === 'utf-16le') { + if (arr.length < 2 || val.length < 2) { + return -1 + } + indexSize = 2; + arrLength /= 2; + valLength /= 2; + byteOffset /= 2; + } } - mul *= 0x80; - - if (val >= mul) val -= Math.pow(2, 8 * byteLength); - return val -}; - -Buffer.prototype.readIntBE = function readIntBE (offset, byteLength, noAssert) { - offset = offset | 0; - byteLength = byteLength | 0; - if (!noAssert) checkOffset(offset, byteLength, this.length); + function read (buf, i) { + if (indexSize === 1) { + return buf[i] + } else { + return buf.readUInt16BE(i * indexSize) + } + } - var i = byteLength; - var mul = 1; - var val = this[offset + --i]; - while (i > 0 && (mul *= 0x100)) { - val += this[offset + --i] * mul; + var i; + if (dir) { + var foundIndex = -1; + for (i = byteOffset; i < arrLength; i++) { + if (read(arr, i) === read(val, foundIndex === -1 ? 0 : i - foundIndex)) { + if (foundIndex === -1) foundIndex = i; + if (i - foundIndex + 1 === valLength) return foundIndex * indexSize + } else { + if (foundIndex !== -1) i -= i - foundIndex; + foundIndex = -1; + } + } + } else { + if (byteOffset + valLength > arrLength) byteOffset = arrLength - valLength; + for (i = byteOffset; i >= 0; i--) { + var found = true; + for (var j = 0; j < valLength; j++) { + if (read(arr, i + j) !== read(val, j)) { + found = false; + break + } + } + if (found) return i + } } - mul *= 0x80; - if (val >= mul) val -= Math.pow(2, 8 * byteLength); + return -1 +} - return val +Buffer.prototype.includes = function includes (val, byteOffset, encoding) { + return this.indexOf(val, byteOffset, encoding) !== -1 }; -Buffer.prototype.readInt8 = function readInt8 (offset, noAssert) { - if (!noAssert) checkOffset(offset, 1, this.length); - if (!(this[offset] & 0x80)) return (this[offset]) - return ((0xff - this[offset] + 1) * -1) +Buffer.prototype.indexOf = function indexOf (val, byteOffset, encoding) { + return bidirectionalIndexOf(this, val, byteOffset, encoding, true) }; -Buffer.prototype.readInt16LE = function readInt16LE (offset, noAssert) { - if (!noAssert) checkOffset(offset, 2, this.length); - var val = this[offset] | (this[offset + 1] << 8); - return (val & 0x8000) ? val | 0xFFFF0000 : val +Buffer.prototype.lastIndexOf = function lastIndexOf (val, byteOffset, encoding) { + return bidirectionalIndexOf(this, val, byteOffset, encoding, false) }; -Buffer.prototype.readInt16BE = function readInt16BE (offset, noAssert) { - if (!noAssert) checkOffset(offset, 2, this.length); - var val = this[offset + 1] | (this[offset] << 8); - return (val & 0x8000) ? val | 0xFFFF0000 : val -}; +function hexWrite (buf, string, offset, length) { + offset = Number(offset) || 0; + var remaining = buf.length - offset; + if (!length) { + length = remaining; + } else { + length = Number(length); + if (length > remaining) { + length = remaining; + } + } -Buffer.prototype.readInt32LE = function readInt32LE (offset, noAssert) { - if (!noAssert) checkOffset(offset, 4, this.length); + // must be an even number of digits + var strLen = string.length; + if (strLen % 2 !== 0) throw new TypeError('Invalid hex string') - return (this[offset]) | - (this[offset + 1] << 8) | - (this[offset + 2] << 16) | - (this[offset + 3] << 24) -}; + if (length > strLen / 2) { + length = strLen / 2; + } + for (var i = 0; i < length; ++i) { + var parsed = parseInt(string.substr(i * 2, 2), 16); + if (isNaN(parsed)) return i + buf[offset + i] = parsed; + } + return i +} -Buffer.prototype.readInt32BE = function readInt32BE (offset, noAssert) { - if (!noAssert) checkOffset(offset, 4, this.length); +function utf8Write (buf, string, offset, length) { + return blitBuffer(utf8ToBytes(string, buf.length - offset), buf, offset, length) +} - return (this[offset] << 24) | - (this[offset + 1] << 16) | - (this[offset + 2] << 8) | - (this[offset + 3]) -}; +function asciiWrite (buf, string, offset, length) { + return blitBuffer(asciiToBytes(string), buf, offset, length) +} -Buffer.prototype.readFloatLE = function readFloatLE (offset, noAssert) { - if (!noAssert) checkOffset(offset, 4, this.length); - return read(this, offset, true, 23, 4) -}; +function latin1Write (buf, string, offset, length) { + return asciiWrite(buf, string, offset, length) +} -Buffer.prototype.readFloatBE = function readFloatBE (offset, noAssert) { - if (!noAssert) checkOffset(offset, 4, this.length); - return read(this, offset, false, 23, 4) -}; - -Buffer.prototype.readDoubleLE = function readDoubleLE (offset, noAssert) { - if (!noAssert) checkOffset(offset, 8, this.length); - return read(this, offset, true, 52, 8) -}; - -Buffer.prototype.readDoubleBE = function readDoubleBE (offset, noAssert) { - if (!noAssert) checkOffset(offset, 8, this.length); - return read(this, offset, false, 52, 8) -}; +function base64Write (buf, string, offset, length) { + return blitBuffer(base64ToBytes(string), buf, offset, length) +} -function checkInt (buf, value, offset, ext, max, min) { - if (!internalIsBuffer(buf)) throw new TypeError('"buffer" argument must be a Buffer instance') - if (value > max || value < min) throw new RangeError('"value" argument is out of bounds') - if (offset + ext > buf.length) throw new RangeError('Index out of range') +function ucs2Write (buf, string, offset, length) { + return blitBuffer(utf16leToBytes(string, buf.length - offset), buf, offset, length) } -Buffer.prototype.writeUIntLE = function writeUIntLE (value, offset, byteLength, noAssert) { - value = +value; - offset = offset | 0; - byteLength = byteLength | 0; - if (!noAssert) { - var maxBytes = Math.pow(2, 8 * byteLength) - 1; - checkInt(this, value, offset, byteLength, maxBytes, 0); +Buffer.prototype.write = function write (string, offset, length, encoding) { + // Buffer#write(string) + if (offset === undefined) { + encoding = 'utf8'; + length = this.length; + offset = 0; + // Buffer#write(string, encoding) + } else if (length === undefined && typeof offset === 'string') { + encoding = offset; + length = this.length; + offset = 0; + // Buffer#write(string, offset[, length][, encoding]) + } else if (isFinite(offset)) { + offset = offset | 0; + if (isFinite(length)) { + length = length | 0; + if (encoding === undefined) encoding = 'utf8'; + } else { + encoding = length; + length = undefined; + } + // legacy write(string, encoding, offset, length) - remove in v0.13 + } else { + throw new Error( + 'Buffer.write(string, encoding, offset[, length]) is no longer supported' + ) } - var mul = 1; - var i = 0; - this[offset] = value & 0xFF; - while (++i < byteLength && (mul *= 0x100)) { - this[offset + i] = (value / mul) & 0xFF; + var remaining = this.length - offset; + if (length === undefined || length > remaining) length = remaining; + + if ((string.length > 0 && (length < 0 || offset < 0)) || offset > this.length) { + throw new RangeError('Attempt to write outside buffer bounds') } - return offset + byteLength -}; + if (!encoding) encoding = 'utf8'; -Buffer.prototype.writeUIntBE = function writeUIntBE (value, offset, byteLength, noAssert) { - value = +value; - offset = offset | 0; - byteLength = byteLength | 0; - if (!noAssert) { - var maxBytes = Math.pow(2, 8 * byteLength) - 1; - checkInt(this, value, offset, byteLength, maxBytes, 0); - } + var loweredCase = false; + for (;;) { + switch (encoding) { + case 'hex': + return hexWrite(this, string, offset, length) - var i = byteLength - 1; - var mul = 1; - this[offset + i] = value & 0xFF; - while (--i >= 0 && (mul *= 0x100)) { - this[offset + i] = (value / mul) & 0xFF; - } + case 'utf8': + case 'utf-8': + return utf8Write(this, string, offset, length) - return offset + byteLength -}; + case 'ascii': + return asciiWrite(this, string, offset, length) -Buffer.prototype.writeUInt8 = function writeUInt8 (value, offset, noAssert) { - value = +value; - offset = offset | 0; - if (!noAssert) checkInt(this, value, offset, 1, 0xff, 0); - if (!Buffer.TYPED_ARRAY_SUPPORT) value = Math.floor(value); - this[offset] = (value & 0xff); - return offset + 1 -}; + case 'latin1': + case 'binary': + return latin1Write(this, string, offset, length) -function objectWriteUInt16 (buf, value, offset, littleEndian) { - if (value < 0) value = 0xffff + value + 1; - for (var i = 0, j = Math.min(buf.length - offset, 2); i < j; ++i) { - buf[offset + i] = (value & (0xff << (8 * (littleEndian ? i : 1 - i)))) >>> - (littleEndian ? i : 1 - i) * 8; - } -} + case 'base64': + // Warning: maxLength not taken into account in base64Write + return base64Write(this, string, offset, length) -Buffer.prototype.writeUInt16LE = function writeUInt16LE (value, offset, noAssert) { - value = +value; - offset = offset | 0; - if (!noAssert) checkInt(this, value, offset, 2, 0xffff, 0); - if (Buffer.TYPED_ARRAY_SUPPORT) { - this[offset] = (value & 0xff); - this[offset + 1] = (value >>> 8); - } else { - objectWriteUInt16(this, value, offset, true); - } - return offset + 2 -}; + case 'ucs2': + case 'ucs-2': + case 'utf16le': + case 'utf-16le': + return ucs2Write(this, string, offset, length) -Buffer.prototype.writeUInt16BE = function writeUInt16BE (value, offset, noAssert) { - value = +value; - offset = offset | 0; - if (!noAssert) checkInt(this, value, offset, 2, 0xffff, 0); - if (Buffer.TYPED_ARRAY_SUPPORT) { - this[offset] = (value >>> 8); - this[offset + 1] = (value & 0xff); - } else { - objectWriteUInt16(this, value, offset, false); + default: + if (loweredCase) throw new TypeError('Unknown encoding: ' + encoding) + encoding = ('' + encoding).toLowerCase(); + loweredCase = true; + } } - return offset + 2 }; -function objectWriteUInt32 (buf, value, offset, littleEndian) { - if (value < 0) value = 0xffffffff + value + 1; - for (var i = 0, j = Math.min(buf.length - offset, 4); i < j; ++i) { - buf[offset + i] = (value >>> (littleEndian ? i : 3 - i) * 8) & 0xff; - } -} - -Buffer.prototype.writeUInt32LE = function writeUInt32LE (value, offset, noAssert) { - value = +value; - offset = offset | 0; - if (!noAssert) checkInt(this, value, offset, 4, 0xffffffff, 0); - if (Buffer.TYPED_ARRAY_SUPPORT) { - this[offset + 3] = (value >>> 24); - this[offset + 2] = (value >>> 16); - this[offset + 1] = (value >>> 8); - this[offset] = (value & 0xff); - } else { - objectWriteUInt32(this, value, offset, true); +Buffer.prototype.toJSON = function toJSON () { + return { + type: 'Buffer', + data: Array.prototype.slice.call(this._arr || this, 0) } - return offset + 4 }; -Buffer.prototype.writeUInt32BE = function writeUInt32BE (value, offset, noAssert) { - value = +value; - offset = offset | 0; - if (!noAssert) checkInt(this, value, offset, 4, 0xffffffff, 0); - if (Buffer.TYPED_ARRAY_SUPPORT) { - this[offset] = (value >>> 24); - this[offset + 1] = (value >>> 16); - this[offset + 2] = (value >>> 8); - this[offset + 3] = (value & 0xff); +function base64Slice (buf, start, end) { + if (start === 0 && end === buf.length) { + return fromByteArray(buf) } else { - objectWriteUInt32(this, value, offset, false); + return fromByteArray(buf.slice(start, end)) } - return offset + 4 -}; +} -Buffer.prototype.writeIntLE = function writeIntLE (value, offset, byteLength, noAssert) { - value = +value; - offset = offset | 0; - if (!noAssert) { - var limit = Math.pow(2, 8 * byteLength - 1); +function utf8Slice (buf, start, end) { + end = Math.min(buf.length, end); + var res = []; - checkInt(this, value, offset, byteLength, limit - 1, -limit); - } + var i = start; + while (i < end) { + var firstByte = buf[i]; + var codePoint = null; + var bytesPerSequence = (firstByte > 0xEF) ? 4 + : (firstByte > 0xDF) ? 3 + : (firstByte > 0xBF) ? 2 + : 1; - var i = 0; - var mul = 1; - var sub = 0; - this[offset] = value & 0xFF; - while (++i < byteLength && (mul *= 0x100)) { - if (value < 0 && sub === 0 && this[offset + i - 1] !== 0) { - sub = 1; - } - this[offset + i] = ((value / mul) >> 0) - sub & 0xFF; - } + if (i + bytesPerSequence <= end) { + var secondByte, thirdByte, fourthByte, tempCodePoint; - return offset + byteLength -}; + switch (bytesPerSequence) { + case 1: + if (firstByte < 0x80) { + codePoint = firstByte; + } + break + case 2: + secondByte = buf[i + 1]; + if ((secondByte & 0xC0) === 0x80) { + tempCodePoint = (firstByte & 0x1F) << 0x6 | (secondByte & 0x3F); + if (tempCodePoint > 0x7F) { + codePoint = tempCodePoint; + } + } + break + case 3: + secondByte = buf[i + 1]; + thirdByte = buf[i + 2]; + if ((secondByte & 0xC0) === 0x80 && (thirdByte & 0xC0) === 0x80) { + tempCodePoint = (firstByte & 0xF) << 0xC | (secondByte & 0x3F) << 0x6 | (thirdByte & 0x3F); + if (tempCodePoint > 0x7FF && (tempCodePoint < 0xD800 || tempCodePoint > 0xDFFF)) { + codePoint = tempCodePoint; + } + } + break + case 4: + secondByte = buf[i + 1]; + thirdByte = buf[i + 2]; + fourthByte = buf[i + 3]; + if ((secondByte & 0xC0) === 0x80 && (thirdByte & 0xC0) === 0x80 && (fourthByte & 0xC0) === 0x80) { + tempCodePoint = (firstByte & 0xF) << 0x12 | (secondByte & 0x3F) << 0xC | (thirdByte & 0x3F) << 0x6 | (fourthByte & 0x3F); + if (tempCodePoint > 0xFFFF && tempCodePoint < 0x110000) { + codePoint = tempCodePoint; + } + } + } + } -Buffer.prototype.writeIntBE = function writeIntBE (value, offset, byteLength, noAssert) { - value = +value; - offset = offset | 0; - if (!noAssert) { - var limit = Math.pow(2, 8 * byteLength - 1); + if (codePoint === null) { + // we did not generate a valid codePoint so insert a + // replacement char (U+FFFD) and advance only 1 byte + codePoint = 0xFFFD; + bytesPerSequence = 1; + } else if (codePoint > 0xFFFF) { + // encode to utf16 (surrogate pair dance) + codePoint -= 0x10000; + res.push(codePoint >>> 10 & 0x3FF | 0xD800); + codePoint = 0xDC00 | codePoint & 0x3FF; + } - checkInt(this, value, offset, byteLength, limit - 1, -limit); + res.push(codePoint); + i += bytesPerSequence; } - var i = byteLength - 1; - var mul = 1; - var sub = 0; - this[offset + i] = value & 0xFF; - while (--i >= 0 && (mul *= 0x100)) { - if (value < 0 && sub === 0 && this[offset + i + 1] !== 0) { - sub = 1; - } - this[offset + i] = ((value / mul) >> 0) - sub & 0xFF; + return decodeCodePointsArray(res) +} + +// Based on http://stackoverflow.com/a/22747272/680742, the browser with +// the lowest limit is Chrome, with 0x10000 args. +// We go 1 magnitude less, for safety +var MAX_ARGUMENTS_LENGTH = 0x1000; + +function decodeCodePointsArray (codePoints) { + var len = codePoints.length; + if (len <= MAX_ARGUMENTS_LENGTH) { + return String.fromCharCode.apply(String, codePoints) // avoid extra slice() } - return offset + byteLength -}; + // Decode in chunks to avoid "call stack size exceeded". + var res = ''; + var i = 0; + while (i < len) { + res += String.fromCharCode.apply( + String, + codePoints.slice(i, i += MAX_ARGUMENTS_LENGTH) + ); + } + return res +} -Buffer.prototype.writeInt8 = function writeInt8 (value, offset, noAssert) { - value = +value; - offset = offset | 0; - if (!noAssert) checkInt(this, value, offset, 1, 0x7f, -0x80); - if (!Buffer.TYPED_ARRAY_SUPPORT) value = Math.floor(value); - if (value < 0) value = 0xff + value + 1; - this[offset] = (value & 0xff); - return offset + 1 -}; +function asciiSlice (buf, start, end) { + var ret = ''; + end = Math.min(buf.length, end); -Buffer.prototype.writeInt16LE = function writeInt16LE (value, offset, noAssert) { - value = +value; - offset = offset | 0; - if (!noAssert) checkInt(this, value, offset, 2, 0x7fff, -0x8000); - if (Buffer.TYPED_ARRAY_SUPPORT) { - this[offset] = (value & 0xff); - this[offset + 1] = (value >>> 8); - } else { - objectWriteUInt16(this, value, offset, true); + for (var i = start; i < end; ++i) { + ret += String.fromCharCode(buf[i] & 0x7F); } - return offset + 2 -}; + return ret +} -Buffer.prototype.writeInt16BE = function writeInt16BE (value, offset, noAssert) { - value = +value; - offset = offset | 0; - if (!noAssert) checkInt(this, value, offset, 2, 0x7fff, -0x8000); - if (Buffer.TYPED_ARRAY_SUPPORT) { - this[offset] = (value >>> 8); - this[offset + 1] = (value & 0xff); - } else { - objectWriteUInt16(this, value, offset, false); +function latin1Slice (buf, start, end) { + var ret = ''; + end = Math.min(buf.length, end); + + for (var i = start; i < end; ++i) { + ret += String.fromCharCode(buf[i]); } - return offset + 2 -}; + return ret +} -Buffer.prototype.writeInt32LE = function writeInt32LE (value, offset, noAssert) { - value = +value; - offset = offset | 0; - if (!noAssert) checkInt(this, value, offset, 4, 0x7fffffff, -0x80000000); - if (Buffer.TYPED_ARRAY_SUPPORT) { - this[offset] = (value & 0xff); - this[offset + 1] = (value >>> 8); - this[offset + 2] = (value >>> 16); - this[offset + 3] = (value >>> 24); - } else { - objectWriteUInt32(this, value, offset, true); +function hexSlice (buf, start, end) { + var len = buf.length; + + if (!start || start < 0) start = 0; + if (!end || end < 0 || end > len) end = len; + + var out = ''; + for (var i = start; i < end; ++i) { + out += toHex(buf[i]); } - return offset + 4 -}; + return out +} -Buffer.prototype.writeInt32BE = function writeInt32BE (value, offset, noAssert) { - value = +value; - offset = offset | 0; - if (!noAssert) checkInt(this, value, offset, 4, 0x7fffffff, -0x80000000); - if (value < 0) value = 0xffffffff + value + 1; +function utf16leSlice (buf, start, end) { + var bytes = buf.slice(start, end); + var res = ''; + for (var i = 0; i < bytes.length; i += 2) { + res += String.fromCharCode(bytes[i] + bytes[i + 1] * 256); + } + return res +} + +Buffer.prototype.slice = function slice (start, end) { + var len = this.length; + start = ~~start; + end = end === undefined ? len : ~~end; + + if (start < 0) { + start += len; + if (start < 0) start = 0; + } else if (start > len) { + start = len; + } + + if (end < 0) { + end += len; + if (end < 0) end = 0; + } else if (end > len) { + end = len; + } + + if (end < start) end = start; + + var newBuf; if (Buffer.TYPED_ARRAY_SUPPORT) { - this[offset] = (value >>> 24); - this[offset + 1] = (value >>> 16); - this[offset + 2] = (value >>> 8); - this[offset + 3] = (value & 0xff); + newBuf = this.subarray(start, end); + newBuf.__proto__ = Buffer.prototype; } else { - objectWriteUInt32(this, value, offset, false); + var sliceLen = end - start; + newBuf = new Buffer(sliceLen, undefined); + for (var i = 0; i < sliceLen; ++i) { + newBuf[i] = this[i + start]; + } } - return offset + 4 + + return newBuf }; -function checkIEEE754 (buf, value, offset, ext, max, min) { - if (offset + ext > buf.length) throw new RangeError('Index out of range') - if (offset < 0) throw new RangeError('Index out of range') +/* + * Need to make sure that buffer isn't trying to write out of bounds. + */ +function checkOffset (offset, ext, length) { + if ((offset % 1) !== 0 || offset < 0) throw new RangeError('offset is not uint') + if (offset + ext > length) throw new RangeError('Trying to access beyond buffer length') } -function writeFloat (buf, value, offset, littleEndian, noAssert) { - if (!noAssert) { - checkIEEE754(buf, value, offset, 4); - } - write(buf, value, offset, littleEndian, 23, 4); - return offset + 4 -} +Buffer.prototype.readUIntLE = function readUIntLE (offset, byteLength, noAssert) { + offset = offset | 0; + byteLength = byteLength | 0; + if (!noAssert) checkOffset(offset, byteLength, this.length); -Buffer.prototype.writeFloatLE = function writeFloatLE (value, offset, noAssert) { - return writeFloat(this, value, offset, true, noAssert) -}; + var val = this[offset]; + var mul = 1; + var i = 0; + while (++i < byteLength && (mul *= 0x100)) { + val += this[offset + i] * mul; + } -Buffer.prototype.writeFloatBE = function writeFloatBE (value, offset, noAssert) { - return writeFloat(this, value, offset, false, noAssert) + return val }; -function writeDouble (buf, value, offset, littleEndian, noAssert) { +Buffer.prototype.readUIntBE = function readUIntBE (offset, byteLength, noAssert) { + offset = offset | 0; + byteLength = byteLength | 0; if (!noAssert) { - checkIEEE754(buf, value, offset, 8); + checkOffset(offset, byteLength, this.length); } - write(buf, value, offset, littleEndian, 52, 8); - return offset + 8 -} -Buffer.prototype.writeDoubleLE = function writeDoubleLE (value, offset, noAssert) { - return writeDouble(this, value, offset, true, noAssert) + var val = this[offset + --byteLength]; + var mul = 1; + while (byteLength > 0 && (mul *= 0x100)) { + val += this[offset + --byteLength] * mul; + } + + return val }; -Buffer.prototype.writeDoubleBE = function writeDoubleBE (value, offset, noAssert) { - return writeDouble(this, value, offset, false, noAssert) +Buffer.prototype.readUInt8 = function readUInt8 (offset, noAssert) { + if (!noAssert) checkOffset(offset, 1, this.length); + return this[offset] }; -// copy(targetBuffer, targetStart=0, sourceStart=0, sourceEnd=buffer.length) -Buffer.prototype.copy = function copy (target, targetStart, start, end) { - if (!start) start = 0; - if (!end && end !== 0) end = this.length; - if (targetStart >= target.length) targetStart = target.length; - if (!targetStart) targetStart = 0; - if (end > 0 && end < start) end = start; +Buffer.prototype.readUInt16LE = function readUInt16LE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 2, this.length); + return this[offset] | (this[offset + 1] << 8) +}; - // Copy 0 bytes; we're done - if (end === start) return 0 - if (target.length === 0 || this.length === 0) return 0 +Buffer.prototype.readUInt16BE = function readUInt16BE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 2, this.length); + return (this[offset] << 8) | this[offset + 1] +}; - // Fatal error conditions - if (targetStart < 0) { - throw new RangeError('targetStart out of bounds') - } - if (start < 0 || start >= this.length) throw new RangeError('sourceStart out of bounds') - if (end < 0) throw new RangeError('sourceEnd out of bounds') +Buffer.prototype.readUInt32LE = function readUInt32LE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 4, this.length); - // Are we oob? - if (end > this.length) end = this.length; - if (target.length - targetStart < end - start) { - end = target.length - targetStart + start; - } + return ((this[offset]) | + (this[offset + 1] << 8) | + (this[offset + 2] << 16)) + + (this[offset + 3] * 0x1000000) +}; - var len = end - start; - var i; +Buffer.prototype.readUInt32BE = function readUInt32BE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 4, this.length); - if (this === target && start < targetStart && targetStart < end) { - // descending copy from end - for (i = len - 1; i >= 0; --i) { - target[i + targetStart] = this[i + start]; - } - } else if (len < 1000 || !Buffer.TYPED_ARRAY_SUPPORT) { - // ascending copy from start - for (i = 0; i < len; ++i) { - target[i + targetStart] = this[i + start]; - } - } else { - Uint8Array.prototype.set.call( - target, - this.subarray(start, start + len), - targetStart - ); + return (this[offset] * 0x1000000) + + ((this[offset + 1] << 16) | + (this[offset + 2] << 8) | + this[offset + 3]) +}; + +Buffer.prototype.readIntLE = function readIntLE (offset, byteLength, noAssert) { + offset = offset | 0; + byteLength = byteLength | 0; + if (!noAssert) checkOffset(offset, byteLength, this.length); + + var val = this[offset]; + var mul = 1; + var i = 0; + while (++i < byteLength && (mul *= 0x100)) { + val += this[offset + i] * mul; } + mul *= 0x80; - return len + if (val >= mul) val -= Math.pow(2, 8 * byteLength); + + return val }; -// Usage: -// buffer.fill(number[, offset[, end]]) -// buffer.fill(buffer[, offset[, end]]) -// buffer.fill(string[, offset[, end]][, encoding]) -Buffer.prototype.fill = function fill (val, start, end, encoding) { - // Handle string cases: - if (typeof val === 'string') { - if (typeof start === 'string') { - encoding = start; - start = 0; - end = this.length; - } else if (typeof end === 'string') { - encoding = end; - end = this.length; - } - if (val.length === 1) { - var code = val.charCodeAt(0); - if (code < 256) { - val = code; - } - } - if (encoding !== undefined && typeof encoding !== 'string') { - throw new TypeError('encoding must be a string') - } - if (typeof encoding === 'string' && !Buffer.isEncoding(encoding)) { - throw new TypeError('Unknown encoding: ' + encoding) - } - } else if (typeof val === 'number') { - val = val & 255; - } +Buffer.prototype.readIntBE = function readIntBE (offset, byteLength, noAssert) { + offset = offset | 0; + byteLength = byteLength | 0; + if (!noAssert) checkOffset(offset, byteLength, this.length); - // Invalid ranges are not set to a default, so can range check early. - if (start < 0 || this.length < start || this.length < end) { - throw new RangeError('Out of range index') + var i = byteLength; + var mul = 1; + var val = this[offset + --i]; + while (i > 0 && (mul *= 0x100)) { + val += this[offset + --i] * mul; } + mul *= 0x80; - if (end <= start) { - return this - } + if (val >= mul) val -= Math.pow(2, 8 * byteLength); - start = start >>> 0; - end = end === undefined ? this.length : end >>> 0; + return val +}; - if (!val) val = 0; +Buffer.prototype.readInt8 = function readInt8 (offset, noAssert) { + if (!noAssert) checkOffset(offset, 1, this.length); + if (!(this[offset] & 0x80)) return (this[offset]) + return ((0xff - this[offset] + 1) * -1) +}; - var i; - if (typeof val === 'number') { - for (i = start; i < end; ++i) { - this[i] = val; - } - } else { - var bytes = internalIsBuffer(val) - ? val - : utf8ToBytes(new Buffer(val, encoding).toString()); - var len = bytes.length; - for (i = 0; i < end - start; ++i) { - this[i + start] = bytes[i % len]; - } - } +Buffer.prototype.readInt16LE = function readInt16LE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 2, this.length); + var val = this[offset] | (this[offset + 1] << 8); + return (val & 0x8000) ? val | 0xFFFF0000 : val +}; - return this +Buffer.prototype.readInt16BE = function readInt16BE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 2, this.length); + var val = this[offset + 1] | (this[offset] << 8); + return (val & 0x8000) ? val | 0xFFFF0000 : val }; -// HELPER FUNCTIONS -// ================ +Buffer.prototype.readInt32LE = function readInt32LE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 4, this.length); -var INVALID_BASE64_RE = /[^+\/0-9A-Za-z-_]/g; + return (this[offset]) | + (this[offset + 1] << 8) | + (this[offset + 2] << 16) | + (this[offset + 3] << 24) +}; -function base64clean (str) { - // Node strips out invalid characters like \n and \t from the string, base64-js does not - str = stringtrim(str).replace(INVALID_BASE64_RE, ''); - // Node converts strings with length < 2 to '' - if (str.length < 2) return '' - // Node allows for non-padded base64 strings (missing trailing ===), base64-js does not - while (str.length % 4 !== 0) { - str = str + '='; - } - return str -} +Buffer.prototype.readInt32BE = function readInt32BE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 4, this.length); -function stringtrim (str) { - if (str.trim) return str.trim() - return str.replace(/^\s+|\s+$/g, '') -} + return (this[offset] << 24) | + (this[offset + 1] << 16) | + (this[offset + 2] << 8) | + (this[offset + 3]) +}; -function toHex (n) { - if (n < 16) return '0' + n.toString(16) - return n.toString(16) -} +Buffer.prototype.readFloatLE = function readFloatLE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 4, this.length); + return read(this, offset, true, 23, 4) +}; -function utf8ToBytes (string, units) { - units = units || Infinity; - var codePoint; - var length = string.length; - var leadSurrogate = null; - var bytes = []; +Buffer.prototype.readFloatBE = function readFloatBE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 4, this.length); + return read(this, offset, false, 23, 4) +}; - for (var i = 0; i < length; ++i) { - codePoint = string.charCodeAt(i); +Buffer.prototype.readDoubleLE = function readDoubleLE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 8, this.length); + return read(this, offset, true, 52, 8) +}; - // is surrogate component - if (codePoint > 0xD7FF && codePoint < 0xE000) { - // last char was a lead - if (!leadSurrogate) { - // no lead yet - if (codePoint > 0xDBFF) { - // unexpected trail - if ((units -= 3) > -1) bytes.push(0xEF, 0xBF, 0xBD); - continue - } else if (i + 1 === length) { - // unpaired lead - if ((units -= 3) > -1) bytes.push(0xEF, 0xBF, 0xBD); - continue - } +Buffer.prototype.readDoubleBE = function readDoubleBE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 8, this.length); + return read(this, offset, false, 52, 8) +}; - // valid lead - leadSurrogate = codePoint; +function checkInt (buf, value, offset, ext, max, min) { + if (!internalIsBuffer(buf)) throw new TypeError('"buffer" argument must be a Buffer instance') + if (value > max || value < min) throw new RangeError('"value" argument is out of bounds') + if (offset + ext > buf.length) throw new RangeError('Index out of range') +} - continue - } +Buffer.prototype.writeUIntLE = function writeUIntLE (value, offset, byteLength, noAssert) { + value = +value; + offset = offset | 0; + byteLength = byteLength | 0; + if (!noAssert) { + var maxBytes = Math.pow(2, 8 * byteLength) - 1; + checkInt(this, value, offset, byteLength, maxBytes, 0); + } - // 2 leads in a row - if (codePoint < 0xDC00) { - if ((units -= 3) > -1) bytes.push(0xEF, 0xBF, 0xBD); - leadSurrogate = codePoint; - continue - } + var mul = 1; + var i = 0; + this[offset] = value & 0xFF; + while (++i < byteLength && (mul *= 0x100)) { + this[offset + i] = (value / mul) & 0xFF; + } - // valid surrogate pair - codePoint = (leadSurrogate - 0xD800 << 10 | codePoint - 0xDC00) + 0x10000; - } else if (leadSurrogate) { - // valid bmp char, but last char was a lead - if ((units -= 3) > -1) bytes.push(0xEF, 0xBF, 0xBD); - } + return offset + byteLength +}; - leadSurrogate = null; +Buffer.prototype.writeUIntBE = function writeUIntBE (value, offset, byteLength, noAssert) { + value = +value; + offset = offset | 0; + byteLength = byteLength | 0; + if (!noAssert) { + var maxBytes = Math.pow(2, 8 * byteLength) - 1; + checkInt(this, value, offset, byteLength, maxBytes, 0); + } - // encode utf8 - if (codePoint < 0x80) { - if ((units -= 1) < 0) break - bytes.push(codePoint); - } else if (codePoint < 0x800) { - if ((units -= 2) < 0) break - bytes.push( - codePoint >> 0x6 | 0xC0, - codePoint & 0x3F | 0x80 - ); - } else if (codePoint < 0x10000) { - if ((units -= 3) < 0) break - bytes.push( - codePoint >> 0xC | 0xE0, - codePoint >> 0x6 & 0x3F | 0x80, - codePoint & 0x3F | 0x80 - ); - } else if (codePoint < 0x110000) { - if ((units -= 4) < 0) break - bytes.push( - codePoint >> 0x12 | 0xF0, - codePoint >> 0xC & 0x3F | 0x80, - codePoint >> 0x6 & 0x3F | 0x80, - codePoint & 0x3F | 0x80 - ); - } else { - throw new Error('Invalid code point') - } + var i = byteLength - 1; + var mul = 1; + this[offset + i] = value & 0xFF; + while (--i >= 0 && (mul *= 0x100)) { + this[offset + i] = (value / mul) & 0xFF; } - return bytes -} - -function asciiToBytes (str) { - var byteArray = []; - for (var i = 0; i < str.length; ++i) { - // Node's code seems to be doing this and not & 0x7F.. - byteArray.push(str.charCodeAt(i) & 0xFF); - } - return byteArray -} + return offset + byteLength +}; -function utf16leToBytes (str, units) { - var c, hi, lo; - var byteArray = []; - for (var i = 0; i < str.length; ++i) { - if ((units -= 2) < 0) break +Buffer.prototype.writeUInt8 = function writeUInt8 (value, offset, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) checkInt(this, value, offset, 1, 0xff, 0); + if (!Buffer.TYPED_ARRAY_SUPPORT) value = Math.floor(value); + this[offset] = (value & 0xff); + return offset + 1 +}; - c = str.charCodeAt(i); - hi = c >> 8; - lo = c % 256; - byteArray.push(lo); - byteArray.push(hi); +function objectWriteUInt16 (buf, value, offset, littleEndian) { + if (value < 0) value = 0xffff + value + 1; + for (var i = 0, j = Math.min(buf.length - offset, 2); i < j; ++i) { + buf[offset + i] = (value & (0xff << (8 * (littleEndian ? i : 1 - i)))) >>> + (littleEndian ? i : 1 - i) * 8; } - - return byteArray -} - - -function base64ToBytes (str) { - return toByteArray(base64clean(str)) } -function blitBuffer (src, dst, offset, length) { - for (var i = 0; i < length; ++i) { - if ((i + offset >= dst.length) || (i >= src.length)) break - dst[i + offset] = src[i]; +Buffer.prototype.writeUInt16LE = function writeUInt16LE (value, offset, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) checkInt(this, value, offset, 2, 0xffff, 0); + if (Buffer.TYPED_ARRAY_SUPPORT) { + this[offset] = (value & 0xff); + this[offset + 1] = (value >>> 8); + } else { + objectWriteUInt16(this, value, offset, true); } - return i -} - -function isnan (val) { - return val !== val // eslint-disable-line no-self-compare -} - - -// the following is from is-buffer, also by Feross Aboukhadijeh and with same lisence -// The _isBuffer check is for Safari 5-7 support, because it's missing -// Object.prototype.constructor. Remove this eventually -function isBuffer(obj) { - return obj != null && (!!obj._isBuffer || isFastBuffer(obj) || isSlowBuffer(obj)) -} + return offset + 2 +}; -function isFastBuffer (obj) { - return !!obj.constructor && typeof obj.constructor.isBuffer === 'function' && obj.constructor.isBuffer(obj) -} +Buffer.prototype.writeUInt16BE = function writeUInt16BE (value, offset, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) checkInt(this, value, offset, 2, 0xffff, 0); + if (Buffer.TYPED_ARRAY_SUPPORT) { + this[offset] = (value >>> 8); + this[offset + 1] = (value & 0xff); + } else { + objectWriteUInt16(this, value, offset, false); + } + return offset + 2 +}; -// For Node v0.10 support. Remove this eventually. -function isSlowBuffer (obj) { - return typeof obj.readFloatLE === 'function' && typeof obj.slice === 'function' && isFastBuffer(obj.slice(0, 0)) +function objectWriteUInt32 (buf, value, offset, littleEndian) { + if (value < 0) value = 0xffffffff + value + 1; + for (var i = 0, j = Math.min(buf.length - offset, 4); i < j; ++i) { + buf[offset + i] = (value >>> (littleEndian ? i : 3 - i) * 8) & 0xff; + } } -var domain; +Buffer.prototype.writeUInt32LE = function writeUInt32LE (value, offset, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) checkInt(this, value, offset, 4, 0xffffffff, 0); + if (Buffer.TYPED_ARRAY_SUPPORT) { + this[offset + 3] = (value >>> 24); + this[offset + 2] = (value >>> 16); + this[offset + 1] = (value >>> 8); + this[offset] = (value & 0xff); + } else { + objectWriteUInt32(this, value, offset, true); + } + return offset + 4 +}; -// This constructor is used to store event handlers. Instantiating this is -// faster than explicitly calling `Object.create(null)` to get a "clean" empty -// object (tested with v8 v4.9). -function EventHandlers() {} -EventHandlers.prototype = Object.create(null); +Buffer.prototype.writeUInt32BE = function writeUInt32BE (value, offset, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) checkInt(this, value, offset, 4, 0xffffffff, 0); + if (Buffer.TYPED_ARRAY_SUPPORT) { + this[offset] = (value >>> 24); + this[offset + 1] = (value >>> 16); + this[offset + 2] = (value >>> 8); + this[offset + 3] = (value & 0xff); + } else { + objectWriteUInt32(this, value, offset, false); + } + return offset + 4 +}; -function EventEmitter() { - EventEmitter.init.call(this); -} +Buffer.prototype.writeIntLE = function writeIntLE (value, offset, byteLength, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) { + var limit = Math.pow(2, 8 * byteLength - 1); -// nodejs oddity -// require('events') === require('events').EventEmitter -EventEmitter.EventEmitter = EventEmitter; + checkInt(this, value, offset, byteLength, limit - 1, -limit); + } -EventEmitter.usingDomains = false; + var i = 0; + var mul = 1; + var sub = 0; + this[offset] = value & 0xFF; + while (++i < byteLength && (mul *= 0x100)) { + if (value < 0 && sub === 0 && this[offset + i - 1] !== 0) { + sub = 1; + } + this[offset + i] = ((value / mul) >> 0) - sub & 0xFF; + } -EventEmitter.prototype.domain = undefined; -EventEmitter.prototype._events = undefined; -EventEmitter.prototype._maxListeners = undefined; + return offset + byteLength +}; -// By default EventEmitters will print a warning if more than 10 listeners are -// added to it. This is a useful default which helps finding memory leaks. -EventEmitter.defaultMaxListeners = 10; +Buffer.prototype.writeIntBE = function writeIntBE (value, offset, byteLength, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) { + var limit = Math.pow(2, 8 * byteLength - 1); -EventEmitter.init = function() { - this.domain = null; - if (EventEmitter.usingDomains) { - // if there is an active domain, then attach to it. - if (domain.active ) ; + checkInt(this, value, offset, byteLength, limit - 1, -limit); } - if (!this._events || this._events === Object.getPrototypeOf(this)._events) { - this._events = new EventHandlers(); - this._eventsCount = 0; + var i = byteLength - 1; + var mul = 1; + var sub = 0; + this[offset + i] = value & 0xFF; + while (--i >= 0 && (mul *= 0x100)) { + if (value < 0 && sub === 0 && this[offset + i + 1] !== 0) { + sub = 1; + } + this[offset + i] = ((value / mul) >> 0) - sub & 0xFF; } - this._maxListeners = this._maxListeners || undefined; + return offset + byteLength }; -// Obviously not all Emitters should be limited to 10. This function allows -// that to be increased. Set to zero for unlimited. -EventEmitter.prototype.setMaxListeners = function setMaxListeners(n) { - if (typeof n !== 'number' || n < 0 || isNaN(n)) - throw new TypeError('"n" argument must be a positive number'); - this._maxListeners = n; - return this; +Buffer.prototype.writeInt8 = function writeInt8 (value, offset, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) checkInt(this, value, offset, 1, 0x7f, -0x80); + if (!Buffer.TYPED_ARRAY_SUPPORT) value = Math.floor(value); + if (value < 0) value = 0xff + value + 1; + this[offset] = (value & 0xff); + return offset + 1 }; -function $getMaxListeners(that) { - if (that._maxListeners === undefined) - return EventEmitter.defaultMaxListeners; - return that._maxListeners; -} - -EventEmitter.prototype.getMaxListeners = function getMaxListeners() { - return $getMaxListeners(this); +Buffer.prototype.writeInt16LE = function writeInt16LE (value, offset, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) checkInt(this, value, offset, 2, 0x7fff, -0x8000); + if (Buffer.TYPED_ARRAY_SUPPORT) { + this[offset] = (value & 0xff); + this[offset + 1] = (value >>> 8); + } else { + objectWriteUInt16(this, value, offset, true); + } + return offset + 2 }; -// These standalone emit* functions are used to optimize calling of event -// handlers for fast cases because emit() itself often has a variable number of -// arguments and can be deoptimized because of that. These functions always have -// the same number of arguments and thus do not get deoptimized, so the code -// inside them can execute faster. -function emitNone(handler, isFn, self) { - if (isFn) - handler.call(self); - else { - var len = handler.length; - var listeners = arrayClone(handler, len); - for (var i = 0; i < len; ++i) - listeners[i].call(self); +Buffer.prototype.writeInt16BE = function writeInt16BE (value, offset, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) checkInt(this, value, offset, 2, 0x7fff, -0x8000); + if (Buffer.TYPED_ARRAY_SUPPORT) { + this[offset] = (value >>> 8); + this[offset + 1] = (value & 0xff); + } else { + objectWriteUInt16(this, value, offset, false); } -} -function emitOne(handler, isFn, self, arg1) { - if (isFn) - handler.call(self, arg1); - else { - var len = handler.length; - var listeners = arrayClone(handler, len); - for (var i = 0; i < len; ++i) - listeners[i].call(self, arg1); + return offset + 2 +}; + +Buffer.prototype.writeInt32LE = function writeInt32LE (value, offset, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) checkInt(this, value, offset, 4, 0x7fffffff, -0x80000000); + if (Buffer.TYPED_ARRAY_SUPPORT) { + this[offset] = (value & 0xff); + this[offset + 1] = (value >>> 8); + this[offset + 2] = (value >>> 16); + this[offset + 3] = (value >>> 24); + } else { + objectWriteUInt32(this, value, offset, true); } -} -function emitTwo(handler, isFn, self, arg1, arg2) { - if (isFn) - handler.call(self, arg1, arg2); - else { - var len = handler.length; - var listeners = arrayClone(handler, len); - for (var i = 0; i < len; ++i) - listeners[i].call(self, arg1, arg2); + return offset + 4 +}; + +Buffer.prototype.writeInt32BE = function writeInt32BE (value, offset, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) checkInt(this, value, offset, 4, 0x7fffffff, -0x80000000); + if (value < 0) value = 0xffffffff + value + 1; + if (Buffer.TYPED_ARRAY_SUPPORT) { + this[offset] = (value >>> 24); + this[offset + 1] = (value >>> 16); + this[offset + 2] = (value >>> 8); + this[offset + 3] = (value & 0xff); + } else { + objectWriteUInt32(this, value, offset, false); } + return offset + 4 +}; + +function checkIEEE754 (buf, value, offset, ext, max, min) { + if (offset + ext > buf.length) throw new RangeError('Index out of range') + if (offset < 0) throw new RangeError('Index out of range') } -function emitThree(handler, isFn, self, arg1, arg2, arg3) { - if (isFn) - handler.call(self, arg1, arg2, arg3); - else { - var len = handler.length; - var listeners = arrayClone(handler, len); - for (var i = 0; i < len; ++i) - listeners[i].call(self, arg1, arg2, arg3); + +function writeFloat (buf, value, offset, littleEndian, noAssert) { + if (!noAssert) { + checkIEEE754(buf, value, offset, 4); } + write(buf, value, offset, littleEndian, 23, 4); + return offset + 4 } -function emitMany(handler, isFn, self, args) { - if (isFn) - handler.apply(self, args); - else { - var len = handler.length; - var listeners = arrayClone(handler, len); - for (var i = 0; i < len; ++i) - listeners[i].apply(self, args); +Buffer.prototype.writeFloatLE = function writeFloatLE (value, offset, noAssert) { + return writeFloat(this, value, offset, true, noAssert) +}; + +Buffer.prototype.writeFloatBE = function writeFloatBE (value, offset, noAssert) { + return writeFloat(this, value, offset, false, noAssert) +}; + +function writeDouble (buf, value, offset, littleEndian, noAssert) { + if (!noAssert) { + checkIEEE754(buf, value, offset, 8); } + write(buf, value, offset, littleEndian, 52, 8); + return offset + 8 } -EventEmitter.prototype.emit = function emit(type) { - var er, handler, len, args, i, events, domain; - var doError = (type === 'error'); +Buffer.prototype.writeDoubleLE = function writeDoubleLE (value, offset, noAssert) { + return writeDouble(this, value, offset, true, noAssert) +}; - events = this._events; - if (events) - doError = (doError && events.error == null); - else if (!doError) - return false; +Buffer.prototype.writeDoubleBE = function writeDoubleBE (value, offset, noAssert) { + return writeDouble(this, value, offset, false, noAssert) +}; - domain = this.domain; +// copy(targetBuffer, targetStart=0, sourceStart=0, sourceEnd=buffer.length) +Buffer.prototype.copy = function copy (target, targetStart, start, end) { + if (!start) start = 0; + if (!end && end !== 0) end = this.length; + if (targetStart >= target.length) targetStart = target.length; + if (!targetStart) targetStart = 0; + if (end > 0 && end < start) end = start; - // If there is no 'error' event listener then throw. - if (doError) { - er = arguments[1]; - if (domain) { - if (!er) - er = new Error('Uncaught, unspecified "error" event'); - er.domainEmitter = this; - er.domain = domain; - er.domainThrown = false; - domain.emit('error', er); - } else if (er instanceof Error) { - throw er; // Unhandled 'error' event - } else { - // At least give some kind of context to the user - var err = new Error('Uncaught, unspecified "error" event. (' + er + ')'); - err.context = er; - throw err; - } - return false; + // Copy 0 bytes; we're done + if (end === start) return 0 + if (target.length === 0 || this.length === 0) return 0 + + // Fatal error conditions + if (targetStart < 0) { + throw new RangeError('targetStart out of bounds') } + if (start < 0 || start >= this.length) throw new RangeError('sourceStart out of bounds') + if (end < 0) throw new RangeError('sourceEnd out of bounds') - handler = events[type]; + // Are we oob? + if (end > this.length) end = this.length; + if (target.length - targetStart < end - start) { + end = target.length - targetStart + start; + } - if (!handler) - return false; + var len = end - start; + var i; - var isFn = typeof handler === 'function'; - len = arguments.length; - switch (len) { - // fast cases - case 1: - emitNone(handler, isFn, this); - break; - case 2: - emitOne(handler, isFn, this, arguments[1]); - break; - case 3: - emitTwo(handler, isFn, this, arguments[1], arguments[2]); - break; - case 4: - emitThree(handler, isFn, this, arguments[1], arguments[2], arguments[3]); - break; - // slower - default: - args = new Array(len - 1); - for (i = 1; i < len; i++) - args[i - 1] = arguments[i]; - emitMany(handler, isFn, this, args); + if (this === target && start < targetStart && targetStart < end) { + // descending copy from end + for (i = len - 1; i >= 0; --i) { + target[i + targetStart] = this[i + start]; + } + } else if (len < 1000 || !Buffer.TYPED_ARRAY_SUPPORT) { + // ascending copy from start + for (i = 0; i < len; ++i) { + target[i + targetStart] = this[i + start]; + } + } else { + Uint8Array.prototype.set.call( + target, + this.subarray(start, start + len), + targetStart + ); } - return true; + return len }; -function _addListener(target, type, listener, prepend) { - var m; - var events; - var existing; - - if (typeof listener !== 'function') - throw new TypeError('"listener" argument must be a function'); +// Usage: +// buffer.fill(number[, offset[, end]]) +// buffer.fill(buffer[, offset[, end]]) +// buffer.fill(string[, offset[, end]][, encoding]) +Buffer.prototype.fill = function fill (val, start, end, encoding) { + // Handle string cases: + if (typeof val === 'string') { + if (typeof start === 'string') { + encoding = start; + start = 0; + end = this.length; + } else if (typeof end === 'string') { + encoding = end; + end = this.length; + } + if (val.length === 1) { + var code = val.charCodeAt(0); + if (code < 256) { + val = code; + } + } + if (encoding !== undefined && typeof encoding !== 'string') { + throw new TypeError('encoding must be a string') + } + if (typeof encoding === 'string' && !Buffer.isEncoding(encoding)) { + throw new TypeError('Unknown encoding: ' + encoding) + } + } else if (typeof val === 'number') { + val = val & 255; + } - events = target._events; - if (!events) { - events = target._events = new EventHandlers(); - target._eventsCount = 0; - } else { - // To avoid recursion in the case that type === "newListener"! Before - // adding it to the listeners, first emit "newListener". - if (events.newListener) { - target.emit('newListener', type, - listener.listener ? listener.listener : listener); + // Invalid ranges are not set to a default, so can range check early. + if (start < 0 || this.length < start || this.length < end) { + throw new RangeError('Out of range index') + } - // Re-assign `events` because a newListener handler could have caused the - // this._events to be assigned to a new object - events = target._events; - } - existing = events[type]; + if (end <= start) { + return this } - if (!existing) { - // Optimize the case of one listener. Don't need the extra array object. - existing = events[type] = listener; - ++target._eventsCount; - } else { - if (typeof existing === 'function') { - // Adding the second element, need to change to array. - existing = events[type] = prepend ? [listener, existing] : - [existing, listener]; - } else { - // If we've already got an array, just append. - if (prepend) { - existing.unshift(listener); - } else { - existing.push(listener); - } - } + start = start >>> 0; + end = end === undefined ? this.length : end >>> 0; - // Check for listener leak - if (!existing.warned) { - m = $getMaxListeners(target); - if (m && m > 0 && existing.length > m) { - existing.warned = true; - var w = new Error('Possible EventEmitter memory leak detected. ' + - existing.length + ' ' + type + ' listeners added. ' + - 'Use emitter.setMaxListeners() to increase limit'); - w.name = 'MaxListenersExceededWarning'; - w.emitter = target; - w.type = type; - w.count = existing.length; - emitWarning(w); - } + if (!val) val = 0; + + var i; + if (typeof val === 'number') { + for (i = start; i < end; ++i) { + this[i] = val; + } + } else { + var bytes = internalIsBuffer(val) + ? val + : utf8ToBytes(new Buffer(val, encoding).toString()); + var len = bytes.length; + for (i = 0; i < end - start; ++i) { + this[i + start] = bytes[i % len]; } } - return target; -} -function emitWarning(e) { - typeof console.warn === 'function' ? console.warn(e) : console.log(e); -} -EventEmitter.prototype.addListener = function addListener(type, listener) { - return _addListener(this, type, listener, false); + return this }; -EventEmitter.prototype.on = EventEmitter.prototype.addListener; +// HELPER FUNCTIONS +// ================ -EventEmitter.prototype.prependListener = - function prependListener(type, listener) { - return _addListener(this, type, listener, true); - }; +var INVALID_BASE64_RE = /[^+\/0-9A-Za-z-_]/g; -function _onceWrap(target, type, listener) { - var fired = false; - function g() { - target.removeListener(type, g); - if (!fired) { - fired = true; - listener.apply(target, arguments); - } +function base64clean (str) { + // Node strips out invalid characters like \n and \t from the string, base64-js does not + str = stringtrim(str).replace(INVALID_BASE64_RE, ''); + // Node converts strings with length < 2 to '' + if (str.length < 2) return '' + // Node allows for non-padded base64 strings (missing trailing ===), base64-js does not + while (str.length % 4 !== 0) { + str = str + '='; } - g.listener = listener; - return g; + return str } -EventEmitter.prototype.once = function once(type, listener) { - if (typeof listener !== 'function') - throw new TypeError('"listener" argument must be a function'); - this.on(type, _onceWrap(this, type, listener)); - return this; -}; - -EventEmitter.prototype.prependOnceListener = - function prependOnceListener(type, listener) { - if (typeof listener !== 'function') - throw new TypeError('"listener" argument must be a function'); - this.prependListener(type, _onceWrap(this, type, listener)); - return this; - }; - -// emits a 'removeListener' event iff the listener was removed -EventEmitter.prototype.removeListener = - function removeListener(type, listener) { - var list, events, position, i, originalListener; - - if (typeof listener !== 'function') - throw new TypeError('"listener" argument must be a function'); - - events = this._events; - if (!events) - return this; - - list = events[type]; - if (!list) - return this; +function stringtrim (str) { + if (str.trim) return str.trim() + return str.replace(/^\s+|\s+$/g, '') +} - if (list === listener || (list.listener && list.listener === listener)) { - if (--this._eventsCount === 0) - this._events = new EventHandlers(); - else { - delete events[type]; - if (events.removeListener) - this.emit('removeListener', type, list.listener || listener); - } - } else if (typeof list !== 'function') { - position = -1; +function toHex (n) { + if (n < 16) return '0' + n.toString(16) + return n.toString(16) +} - for (i = list.length; i-- > 0;) { - if (list[i] === listener || - (list[i].listener && list[i].listener === listener)) { - originalListener = list[i].listener; - position = i; - break; - } - } +function utf8ToBytes (string, units) { + units = units || Infinity; + var codePoint; + var length = string.length; + var leadSurrogate = null; + var bytes = []; - if (position < 0) - return this; + for (var i = 0; i < length; ++i) { + codePoint = string.charCodeAt(i); - if (list.length === 1) { - list[0] = undefined; - if (--this._eventsCount === 0) { - this._events = new EventHandlers(); - return this; - } else { - delete events[type]; - } - } else { - spliceOne(list, position); + // is surrogate component + if (codePoint > 0xD7FF && codePoint < 0xE000) { + // last char was a lead + if (!leadSurrogate) { + // no lead yet + if (codePoint > 0xDBFF) { + // unexpected trail + if ((units -= 3) > -1) bytes.push(0xEF, 0xBF, 0xBD); + continue + } else if (i + 1 === length) { + // unpaired lead + if ((units -= 3) > -1) bytes.push(0xEF, 0xBF, 0xBD); + continue } - if (events.removeListener) - this.emit('removeListener', type, originalListener || listener); - } - - return this; - }; - -EventEmitter.prototype.removeAllListeners = - function removeAllListeners(type) { - var listeners, events; - - events = this._events; - if (!events) - return this; + // valid lead + leadSurrogate = codePoint; - // not listening for removeListener, no need to emit - if (!events.removeListener) { - if (arguments.length === 0) { - this._events = new EventHandlers(); - this._eventsCount = 0; - } else if (events[type]) { - if (--this._eventsCount === 0) - this._events = new EventHandlers(); - else - delete events[type]; - } - return this; + continue } - // emit removeListener for all listeners on all events - if (arguments.length === 0) { - var keys = Object.keys(events); - for (var i = 0, key; i < keys.length; ++i) { - key = keys[i]; - if (key === 'removeListener') continue; - this.removeAllListeners(key); - } - this.removeAllListeners('removeListener'); - this._events = new EventHandlers(); - this._eventsCount = 0; - return this; + // 2 leads in a row + if (codePoint < 0xDC00) { + if ((units -= 3) > -1) bytes.push(0xEF, 0xBF, 0xBD); + leadSurrogate = codePoint; + continue } - listeners = events[type]; + // valid surrogate pair + codePoint = (leadSurrogate - 0xD800 << 10 | codePoint - 0xDC00) + 0x10000; + } else if (leadSurrogate) { + // valid bmp char, but last char was a lead + if ((units -= 3) > -1) bytes.push(0xEF, 0xBF, 0xBD); + } - if (typeof listeners === 'function') { - this.removeListener(type, listeners); - } else if (listeners) { - // LIFO order - do { - this.removeListener(type, listeners[listeners.length - 1]); - } while (listeners[0]); - } + leadSurrogate = null; - return this; - }; + // encode utf8 + if (codePoint < 0x80) { + if ((units -= 1) < 0) break + bytes.push(codePoint); + } else if (codePoint < 0x800) { + if ((units -= 2) < 0) break + bytes.push( + codePoint >> 0x6 | 0xC0, + codePoint & 0x3F | 0x80 + ); + } else if (codePoint < 0x10000) { + if ((units -= 3) < 0) break + bytes.push( + codePoint >> 0xC | 0xE0, + codePoint >> 0x6 & 0x3F | 0x80, + codePoint & 0x3F | 0x80 + ); + } else if (codePoint < 0x110000) { + if ((units -= 4) < 0) break + bytes.push( + codePoint >> 0x12 | 0xF0, + codePoint >> 0xC & 0x3F | 0x80, + codePoint >> 0x6 & 0x3F | 0x80, + codePoint & 0x3F | 0x80 + ); + } else { + throw new Error('Invalid code point') + } + } -EventEmitter.prototype.listeners = function listeners(type) { - var evlistener; - var ret; - var events = this._events; + return bytes +} - if (!events) - ret = []; - else { - evlistener = events[type]; - if (!evlistener) - ret = []; - else if (typeof evlistener === 'function') - ret = [evlistener.listener || evlistener]; - else - ret = unwrapListeners(evlistener); +function asciiToBytes (str) { + var byteArray = []; + for (var i = 0; i < str.length; ++i) { + // Node's code seems to be doing this and not & 0x7F.. + byteArray.push(str.charCodeAt(i) & 0xFF); } + return byteArray +} - return ret; -}; +function utf16leToBytes (str, units) { + var c, hi, lo; + var byteArray = []; + for (var i = 0; i < str.length; ++i) { + if ((units -= 2) < 0) break -EventEmitter.listenerCount = function(emitter, type) { - if (typeof emitter.listenerCount === 'function') { - return emitter.listenerCount(type); - } else { - return listenerCount$1.call(emitter, type); + c = str.charCodeAt(i); + hi = c >> 8; + lo = c % 256; + byteArray.push(lo); + byteArray.push(hi); } -}; -EventEmitter.prototype.listenerCount = listenerCount$1; -function listenerCount$1(type) { - var events = this._events; + return byteArray +} - if (events) { - var evlistener = events[type]; - if (typeof evlistener === 'function') { - return 1; - } else if (evlistener) { - return evlistener.length; - } +function base64ToBytes (str) { + return toByteArray(base64clean(str)) +} + +function blitBuffer (src, dst, offset, length) { + for (var i = 0; i < length; ++i) { + if ((i + offset >= dst.length) || (i >= src.length)) break + dst[i + offset] = src[i]; } + return i +} - return 0; +function isnan (val) { + return val !== val // eslint-disable-line no-self-compare } -EventEmitter.prototype.eventNames = function eventNames() { - return this._eventsCount > 0 ? Reflect.ownKeys(this._events) : []; -}; -// About 1.5x faster than the two-arg version of Array#splice(). -function spliceOne(list, index) { - for (var i = index, k = i + 1, n = list.length; k < n; i += 1, k += 1) - list[i] = list[k]; - list.pop(); +// the following is from is-buffer, also by Feross Aboukhadijeh and with same lisence +// The _isBuffer check is for Safari 5-7 support, because it's missing +// Object.prototype.constructor. Remove this eventually +function isBuffer(obj) { + return obj != null && (!!obj._isBuffer || isFastBuffer(obj) || isSlowBuffer(obj)) } -function arrayClone(arr, i) { - var copy = new Array(i); - while (i--) - copy[i] = arr[i]; - return copy; +function isFastBuffer (obj) { + return !!obj.constructor && typeof obj.constructor.isBuffer === 'function' && obj.constructor.isBuffer(obj) } -function unwrapListeners(arr) { - var ret = new Array(arr.length); - for (var i = 0; i < ret.length; ++i) { - ret[i] = arr[i].listener || arr[i]; - } - return ret; +// For Node v0.10 support. Remove this eventually. +function isSlowBuffer (obj) { + return typeof obj.readFloatLE === 'function' && typeof obj.slice === 'function' && isFastBuffer(obj.slice(0, 0)) } // shim for using process in browser @@ -2627,7 +2627,7 @@ function format(f) { } }); for (var x = args[i]; i < len; x = args[++i]) { - if (isNull(x) || !isObject$1(x)) { + if (isNull(x) || !isObject(x)) { str += ' ' + x; } else { str += ' ' + inspect(x); @@ -3035,19 +3035,19 @@ function isUndefined(arg) { } function isRegExp(re) { - return isObject$1(re) && objectToString(re) === '[object RegExp]'; + return isObject(re) && objectToString(re) === '[object RegExp]'; } -function isObject$1(arg) { +function isObject(arg) { return typeof arg === 'object' && arg !== null; } function isDate(d) { - return isObject$1(d) && objectToString(d) === '[object Date]'; + return isObject(d) && objectToString(d) === '[object Date]'; } function isError(e) { - return isObject$1(e) && + return isObject(e) && (objectToString(e) === '[object Error]' || e instanceof Error); } @@ -3061,7 +3061,7 @@ function objectToString(o) { function _extend(origin, add) { // Don't do anything if add isn't an object - if (!add || !isObject$1(add)) return origin; + if (!add || !isObject(add)) return origin; var keys = Object.keys(add); var i = keys.length; @@ -4966,8 +4966,6 @@ Stream.prototype.pipe = function(dest, options) { return dest; }; -const bom_utf8 = Buffer.from([239, 187, 191]); - class CsvError extends Error { constructor(code, message, ...contexts) { if(Array.isArray(message)) message = message.join(' '); @@ -4985,16 +4983,10 @@ class CsvError extends Error { } } -const isObject = function(obj){ +const is_object = function(obj){ return typeof obj === 'object' && obj !== null && ! Array.isArray(obj); }; -const underscore = function(str){ - return str.replace(/([A-Z])/g, function(_, match){ - return '_' + match.toLowerCase(); - }); -}; - // Lodash implementation of `get` const charCodeOfDot = '.'.charCodeAt(0); @@ -5072,36 +5064,435 @@ const get = function(object, path){ return (index && index === length) ? object : undefined; }; +const normalize_columns = function(columns){ + if(columns === undefined || columns === null){ + return [undefined, undefined]; + } + if(typeof columns !== 'object'){ + return [Error('Invalid option "columns": expect an array or an object')]; + } + if(!Array.isArray(columns)){ + const newcolumns = []; + for(const k in columns){ + newcolumns.push({ + key: k, + header: columns[k] + }); + } + columns = newcolumns; + }else { + const newcolumns = []; + for(const column of columns){ + if(typeof column === 'string'){ + newcolumns.push({ + key: column, + header: column + }); + }else if(typeof column === 'object' && column !== null && !Array.isArray(column)){ + if(!column.key){ + return [Error('Invalid column definition: property "key" is required')]; + } + if(column.header === undefined){ + column.header = column.key; + } + newcolumns.push(column); + }else { + return [Error('Invalid column definition: expect a string or an object')]; + } + } + columns = newcolumns; + } + return [undefined, columns]; +}; + +const underscore = function(str){ + return str.replace(/([A-Z])/g, function(_, match){ + return '_' + match.toLowerCase(); + }); +}; + +const normalize_options = function(opts) { + const options = {}; + // Merge with user options + for(const opt in opts){ + options[underscore(opt)] = opts[opt]; + } + // Normalize option `bom` + if(options.bom === undefined || options.bom === null || options.bom === false){ + options.bom = false; + }else if(options.bom !== true){ + return [new CsvError('CSV_OPTION_BOOLEAN_INVALID_TYPE', [ + 'option `bom` is optional and must be a boolean value,', + `got ${JSON.stringify(options.bom)}` + ])]; + } + // Normalize option `delimiter` + if(options.delimiter === undefined || options.delimiter === null){ + options.delimiter = ','; + }else if(isBuffer(options.delimiter)){ + options.delimiter = options.delimiter.toString(); + }else if(typeof options.delimiter !== 'string'){ + return [new CsvError('CSV_OPTION_DELIMITER_INVALID_TYPE', [ + 'option `delimiter` must be a buffer or a string,', + `got ${JSON.stringify(options.delimiter)}` + ])]; + } + // Normalize option `quote` + if(options.quote === undefined || options.quote === null){ + options.quote = '"'; + }else if(options.quote === true){ + options.quote = '"'; + }else if(options.quote === false){ + options.quote = ''; + }else if (isBuffer(options.quote)){ + options.quote = options.quote.toString(); + }else if(typeof options.quote !== 'string'){ + return [new CsvError('CSV_OPTION_QUOTE_INVALID_TYPE', [ + 'option `quote` must be a boolean, a buffer or a string,', + `got ${JSON.stringify(options.quote)}` + ])]; + } + // Normalize option `quoted` + if(options.quoted === undefined || options.quoted === null){ + options.quoted = false; + } + // Normalize option `quoted_empty` + if(options.quoted_empty === undefined || options.quoted_empty === null){ + options.quoted_empty = undefined; + } + // Normalize option `quoted_match` + if(options.quoted_match === undefined || options.quoted_match === null || options.quoted_match === false){ + options.quoted_match = null; + }else if(!Array.isArray(options.quoted_match)){ + options.quoted_match = [options.quoted_match]; + } + if(options.quoted_match){ + for(const quoted_match of options.quoted_match){ + const isString = typeof quoted_match === 'string'; + const isRegExp = quoted_match instanceof RegExp; + if(!isString && !isRegExp){ + return [Error(`Invalid Option: quoted_match must be a string or a regex, got ${JSON.stringify(quoted_match)}`)]; + } + } + } + // Normalize option `quoted_string` + if(options.quoted_string === undefined || options.quoted_string === null){ + options.quoted_string = false; + } + // Normalize option `eof` + if(options.eof === undefined || options.eof === null){ + options.eof = true; + } + // Normalize option `escape` + if(options.escape === undefined || options.escape === null){ + options.escape = '"'; + }else if(isBuffer(options.escape)){ + options.escape = options.escape.toString(); + }else if(typeof options.escape !== 'string'){ + return [Error(`Invalid Option: escape must be a buffer or a string, got ${JSON.stringify(options.escape)}`)]; + } + if (options.escape.length > 1){ + return [Error(`Invalid Option: escape must be one character, got ${options.escape.length} characters`)]; + } + // Normalize option `header` + if(options.header === undefined || options.header === null){ + options.header = false; + } + // Normalize option `columns` + const [errColumns, columns] = normalize_columns(options.columns); + if(errColumns !== undefined) return [errColumns]; + options.columns = columns; + // Normalize option `quoted` + if(options.quoted === undefined || options.quoted === null){ + options.quoted = false; + } + // Normalize option `cast` + if(options.cast === undefined || options.cast === null){ + options.cast = {}; + } + // Normalize option cast.bigint + if(options.cast.bigint === undefined || options.cast.bigint === null){ + // Cast boolean to string by default + options.cast.bigint = value => '' + value; + } + // Normalize option cast.boolean + if(options.cast.boolean === undefined || options.cast.boolean === null){ + // Cast boolean to string by default + options.cast.boolean = value => value ? '1' : ''; + } + // Normalize option cast.date + if(options.cast.date === undefined || options.cast.date === null){ + // Cast date to timestamp string by default + options.cast.date = value => '' + value.getTime(); + } + // Normalize option cast.number + if(options.cast.number === undefined || options.cast.number === null){ + // Cast number to string using native casting by default + options.cast.number = value => '' + value; + } + // Normalize option cast.object + if(options.cast.object === undefined || options.cast.object === null){ + // Stringify object as JSON by default + options.cast.object = value => JSON.stringify(value); + } + // Normalize option cast.string + if(options.cast.string === undefined || options.cast.string === null){ + // Leave string untouched + options.cast.string = function(value){return value;}; + } + // Normalize option `on_record` + if(options.on_record !== undefined && typeof options.on_record !== 'function'){ + return [Error(`Invalid Option: "on_record" must be a function.`)]; + } + // Normalize option `record_delimiter` + if(options.record_delimiter === undefined || options.record_delimiter === null){ + options.record_delimiter = '\n'; + }else if(isBuffer(options.record_delimiter)){ + options.record_delimiter = options.record_delimiter.toString(); + }else if(typeof options.record_delimiter !== 'string'){ + return [Error(`Invalid Option: record_delimiter must be a buffer or a string, got ${JSON.stringify(options.record_delimiter)}`)]; + } + switch(options.record_delimiter){ + case 'auto': + options.record_delimiter = null; + break; + case 'unix': + options.record_delimiter = "\n"; + break; + case 'mac': + options.record_delimiter = "\r"; + break; + case 'windows': + options.record_delimiter = "\r\n"; + break; + case 'ascii': + options.record_delimiter = "\u001e"; + break; + case 'unicode': + options.record_delimiter = "\u2028"; + break; + } + return [undefined, options]; +}; + +const bom_utf8 = Buffer.from([239, 187, 191]); + +const stringifier = function(options, state, info){ + return { + options: options, + state: state, + info: info, + __transform: function(chunk, push){ + // Chunk validation + if(!Array.isArray(chunk) && typeof chunk !== 'object'){ + return Error(`Invalid Record: expect an array or an object, got ${JSON.stringify(chunk)}`); + } + // Detect columns from the first record + if(this.info.records === 0){ + if(Array.isArray(chunk)){ + if(this.options.header === true && this.options.columns === undefined){ + return Error('Undiscoverable Columns: header option requires column option or object records'); + } + }else if(this.options.columns === undefined){ + const [err, columns] = normalize_columns(Object.keys(chunk)); + if(err) return; + this.options.columns = columns; + } + } + // Emit the header + if(this.info.records === 0){ + this.bom(push); + const err = this.headers(push); + if(err) return err; + } + // Emit and stringify the record if an object or an array + try{ + // this.emit('record', chunk, this.info.records); + if(this.options.on_record){ + this.options.on_record(chunk, this.info.records); + } + }catch(err){ + return err; + } + // Convert the record into a string + let err, chunk_string; + if(this.options.eof){ + [err, chunk_string] = this.stringify(chunk); + if(err) return err; + if(chunk_string === undefined){ + return; + }else { + chunk_string = chunk_string + this.options.record_delimiter; + } + }else { + [err, chunk_string] = this.stringify(chunk); + if(err) return err; + if(chunk_string === undefined){ + return; + }else { + if(this.options.header || this.info.records){ + chunk_string = this.options.record_delimiter + chunk_string; + } + } + } + // Emit the csv + this.info.records++; + push(chunk_string); + }, + stringify: function(chunk, chunkIsHeader=false){ + if(typeof chunk !== 'object'){ + return [undefined, chunk]; + } + const {columns} = this.options; + const record = []; + // Record is an array + if(Array.isArray(chunk)){ + // We are getting an array but the user has specified output columns. In + // this case, we respect the columns indexes + if(columns){ + chunk.splice(columns.length); + } + // Cast record elements + for(let i=0; i= 0; + const containsQuote = (quote !== '') && value.indexOf(quote) >= 0; + const containsEscape = value.indexOf(escape) >= 0 && (escape !== quote); + const containsRecordDelimiter = value.indexOf(record_delimiter) >= 0; + const quotedString = quoted_string && typeof field === 'string'; + let quotedMatch = quoted_match && quoted_match.filter(quoted_match => { + if(typeof quoted_match === 'string'){ + return value.indexOf(quoted_match) !== -1; + }else { + return quoted_match.test(value); + } + }); + quotedMatch = quotedMatch && quotedMatch.length > 0; + const shouldQuote = containsQuote === true || containsdelimiter || containsRecordDelimiter || quoted || quotedString || quotedMatch; + if(shouldQuote === true && containsEscape === true){ + const regexp = escape === '\\' + ? new RegExp(escape + escape, 'g') + : new RegExp(escape, 'g'); + value = value.replace(regexp, escape + escape); + } + if(containsQuote === true){ + const regexp = new RegExp(quote,'g'); + value = value.replace(regexp, escape + quote); + } + if(shouldQuote === true){ + value = quote + value + quote; + } + csvrecord += value; + }else if(quoted_empty === true || (field === '' && quoted_string === true && quoted_empty !== false)){ + csvrecord += quote + quote; + } + if(i !== record.length - 1){ + csvrecord += delimiter; + } + } + return [undefined, csvrecord]; + }, + bom: function(push){ + if(this.options.bom !== true){ + return; + } + push(bom_utf8); + }, + headers: function(push){ + if(this.options.header === false){ + return; + } + if(this.options.columns === undefined){ + return; + } + let err; + let headers = this.options.columns.map(column => column.header); + if(this.options.eof){ + [err, headers] = this.stringify(headers, true); + headers += this.options.record_delimiter; + }else { + [err, headers] = this.stringify(headers); + } + if(err) return err; + push(headers); + }, + __cast: function(value, context){ + const type = typeof value; + try{ + if(type === 'string'){ // Fine for 99% of the cases + return [undefined, this.options.cast.string(value, context)]; + }else if(type === 'bigint'){ + return [undefined, this.options.cast.bigint(value, context)]; + }else if(type === 'number'){ + return [undefined, this.options.cast.number(value, context)]; + }else if(type === 'boolean'){ + return [undefined, this.options.cast.boolean(value, context)]; + }else if(value instanceof Date){ + return [undefined, this.options.cast.date(value, context)]; + }else if(type === 'object' && value !== null){ + return [undefined, this.options.cast.object(value, context)]; + }else { + return [undefined, value, value]; + } + }catch(err){ + return [err]; + } + } + }; +}; + class Stringifier extends Transform { constructor(opts = {}){ super({...{writableObjectMode: true}, ...opts}); - const options = {}; - let err; - // Merge with user options - for(const opt in opts){ - options[underscore(opt)] = opts[opt]; - } - if((err = this.normalize(options)) !== undefined) throw err; - switch(options.record_delimiter){ - case 'auto': - options.record_delimiter = null; - break; - case 'unix': - options.record_delimiter = "\n"; - break; - case 'mac': - options.record_delimiter = "\r"; - break; - case 'windows': - options.record_delimiter = "\r\n"; - break; - case 'ascii': - options.record_delimiter = "\u001e"; - break; - case 'unicode': - options.record_delimiter = "\u2028"; - break; - } + const [err, options] = normalize_options(opts); + if(err !== undefined) throw err; // Expose options this.options = options; // Internal state @@ -5112,145 +5503,16 @@ class Stringifier extends Transform { this.info = { records: 0 }; - } - normalize(options){ - // Normalize option `bom` - if(options.bom === undefined || options.bom === null || options.bom === false){ - options.bom = false; - }else if(options.bom !== true){ - return new CsvError('CSV_OPTION_BOOLEAN_INVALID_TYPE', [ - 'option `bom` is optional and must be a boolean value,', - `got ${JSON.stringify(options.bom)}` - ]); - } - // Normalize option `delimiter` - if(options.delimiter === undefined || options.delimiter === null){ - options.delimiter = ','; - }else if(isBuffer(options.delimiter)){ - options.delimiter = options.delimiter.toString(); - }else if(typeof options.delimiter !== 'string'){ - return new CsvError('CSV_OPTION_DELIMITER_INVALID_TYPE', [ - 'option `delimiter` must be a buffer or a string,', - `got ${JSON.stringify(options.delimiter)}` - ]); - } - // Normalize option `quote` - if(options.quote === undefined || options.quote === null){ - options.quote = '"'; - }else if(options.quote === true){ - options.quote = '"'; - }else if(options.quote === false){ - options.quote = ''; - }else if (isBuffer(options.quote)){ - options.quote = options.quote.toString(); - }else if(typeof options.quote !== 'string'){ - return new CsvError('CSV_OPTION_QUOTE_INVALID_TYPE', [ - 'option `quote` must be a boolean, a buffer or a string,', - `got ${JSON.stringify(options.quote)}` - ]); - } - // Normalize option `quoted` - if(options.quoted === undefined || options.quoted === null){ - options.quoted = false; - } - // Normalize option `quoted_empty` - if(options.quoted_empty === undefined || options.quoted_empty === null){ - options.quoted_empty = undefined; - } - // Normalize option `quoted_match` - if(options.quoted_match === undefined || options.quoted_match === null || options.quoted_match === false){ - options.quoted_match = null; - }else if(!Array.isArray(options.quoted_match)){ - options.quoted_match = [options.quoted_match]; - } - if(options.quoted_match){ - for(const quoted_match of options.quoted_match){ - const isString = typeof quoted_match === 'string'; - const isRegExp = quoted_match instanceof RegExp; - if(!isString && !isRegExp){ - return Error(`Invalid Option: quoted_match must be a string or a regex, got ${JSON.stringify(quoted_match)}`); - } - } - } - // Normalize option `quoted_string` - if(options.quoted_string === undefined || options.quoted_string === null){ - options.quoted_string = false; - } - // Normalize option `eof` - if(options.eof === undefined || options.eof === null){ - options.eof = true; - } - // Normalize option `escape` - if(options.escape === undefined || options.escape === null){ - options.escape = '"'; - }else if(isBuffer(options.escape)){ - options.escape = options.escape.toString(); - }else if(typeof options.escape !== 'string'){ - return Error(`Invalid Option: escape must be a buffer or a string, got ${JSON.stringify(options.escape)}`); - } - if (options.escape.length > 1){ - return Error(`Invalid Option: escape must be one character, got ${options.escape.length} characters`); - } - // Normalize option `header` - if(options.header === undefined || options.header === null){ - options.header = false; - } - // Normalize option `columns` - const [err, columns] = this.normalize_columns(options.columns); - if(err) return err; - options.columns = columns; - // Normalize option `quoted` - if(options.quoted === undefined || options.quoted === null){ - options.quoted = false; - } - // Normalize option `cast` - if(options.cast === undefined || options.cast === null){ - options.cast = {}; - } - // Normalize option cast.bigint - if(options.cast.bigint === undefined || options.cast.bigint === null){ - // Cast boolean to string by default - options.cast.bigint = value => '' + value; - } - // Normalize option cast.boolean - if(options.cast.boolean === undefined || options.cast.boolean === null){ - // Cast boolean to string by default - options.cast.boolean = value => value ? '1' : ''; - } - // Normalize option cast.date - if(options.cast.date === undefined || options.cast.date === null){ - // Cast date to timestamp string by default - options.cast.date = value => '' + value.getTime(); - } - // Normalize option cast.number - if(options.cast.number === undefined || options.cast.number === null){ - // Cast number to string using native casting by default - options.cast.number = value => '' + value; - } - // Normalize option cast.object - if(options.cast.object === undefined || options.cast.object === null){ - // Stringify object as JSON by default - options.cast.object = value => JSON.stringify(value); - } - // Normalize option cast.string - if(options.cast.string === undefined || options.cast.string === null){ - // Leave string untouched - options.cast.string = function(value){return value;}; - } - // Normalize option `record_delimiter` - if(options.record_delimiter === undefined || options.record_delimiter === null){ - options.record_delimiter = '\n'; - }else if(isBuffer(options.record_delimiter)){ - options.record_delimiter = options.record_delimiter.toString(); - }else if(typeof options.record_delimiter !== 'string'){ - return Error(`Invalid Option: record_delimiter must be a buffer or a string, got ${JSON.stringify(options.record_delimiter)}`); - } + this.api = stringifier(this.options, this.state, this.info); + this.api.options.on_record = (...args) => { + this.emit('record', ...args); + }; } _transform(chunk, encoding, callback){ if(this.state.stop === true){ return; } - const err = this.__transform(chunk); + const err = this.api.__transform(chunk, this.push.bind(this)); if(err !== undefined){ this.state.stop = true; } @@ -5263,251 +5525,12 @@ class Stringifier extends Transform { return; } if(this.info.records === 0){ - this.bom(); - const err = this.headers(); + this.api.bom(this.push.bind(this)); + const err = this.api.headers(this.push.bind(this)); if(err) callback(err); } callback(); } - __transform(chunk){ - // Chunk validation - if(!Array.isArray(chunk) && typeof chunk !== 'object'){ - return Error(`Invalid Record: expect an array or an object, got ${JSON.stringify(chunk)}`); - } - // Detect columns from the first record - if(this.info.records === 0){ - if(Array.isArray(chunk)){ - if(this.options.header === true && this.options.columns === undefined){ - return Error('Undiscoverable Columns: header option requires column option or object records'); - } - }else if(this.options.columns === undefined){ - const [err, columns] = this.normalize_columns(Object.keys(chunk)); - if(err) return; - this.options.columns = columns; - } - } - // Emit the header - if(this.info.records === 0){ - this.bom(); - const err = this.headers(); - if(err) return err; - } - // Emit and stringify the record if an object or an array - try{ - this.emit('record', chunk, this.info.records); - }catch(err){ - return err; - } - // Convert the record into a string - let err, chunk_string; - if(this.options.eof){ - [err, chunk_string] = this.stringify(chunk); - if(err) return err; - if(chunk_string === undefined){ - return; - }else { - chunk_string = chunk_string + this.options.record_delimiter; - } - }else { - [err, chunk_string] = this.stringify(chunk); - if(err) return err; - if(chunk_string === undefined){ - return; - }else { - if(this.options.header || this.info.records){ - chunk_string = this.options.record_delimiter + chunk_string; - } - } - } - // Emit the csv - this.info.records++; - this.push(chunk_string); - } - stringify(chunk, chunkIsHeader=false){ - if(typeof chunk !== 'object'){ - return [undefined, chunk]; - } - const {columns} = this.options; - const record = []; - // Record is an array - if(Array.isArray(chunk)){ - // We are getting an array but the user has specified output columns. In - // this case, we respect the columns indexes - if(columns){ - chunk.splice(columns.length); - } - // Cast record elements - for(let i=0; i= 0; - const containsQuote = (quote !== '') && value.indexOf(quote) >= 0; - const containsEscape = value.indexOf(escape) >= 0 && (escape !== quote); - const containsRecordDelimiter = value.indexOf(record_delimiter) >= 0; - const quotedString = quoted_string && typeof field === 'string'; - let quotedMatch = quoted_match && quoted_match.filter(quoted_match => { - if(typeof quoted_match === 'string'){ - return value.indexOf(quoted_match) !== -1; - }else { - return quoted_match.test(value); - } - }); - quotedMatch = quotedMatch && quotedMatch.length > 0; - const shouldQuote = containsQuote === true || containsdelimiter || containsRecordDelimiter || quoted || quotedString || quotedMatch; - if(shouldQuote === true && containsEscape === true){ - const regexp = escape === '\\' - ? new RegExp(escape + escape, 'g') - : new RegExp(escape, 'g'); - value = value.replace(regexp, escape + escape); - } - if(containsQuote === true){ - const regexp = new RegExp(quote,'g'); - value = value.replace(regexp, escape + quote); - } - if(shouldQuote === true){ - value = quote + value + quote; - } - csvrecord += value; - }else if(quoted_empty === true || (field === '' && quoted_string === true && quoted_empty !== false)){ - csvrecord += quote + quote; - } - if(i !== record.length - 1){ - csvrecord += delimiter; - } - } - return [undefined, csvrecord]; - } - bom(){ - if(this.options.bom !== true){ - return; - } - this.push(bom_utf8); - } - headers(){ - if(this.options.header === false){ - return; - } - if(this.options.columns === undefined){ - return; - } - let err; - let headers = this.options.columns.map(column => column.header); - if(this.options.eof){ - [err, headers] = this.stringify(headers, true); - headers += this.options.record_delimiter; - }else { - [err, headers] = this.stringify(headers); - } - if(err) return err; - this.push(headers); - } - __cast(value, context){ - const type = typeof value; - try{ - if(type === 'string'){ // Fine for 99% of the cases - return [undefined, this.options.cast.string(value, context)]; - }else if(type === 'bigint'){ - return [undefined, this.options.cast.bigint(value, context)]; - }else if(type === 'number'){ - return [undefined, this.options.cast.number(value, context)]; - }else if(type === 'boolean'){ - return [undefined, this.options.cast.boolean(value, context)]; - }else if(value instanceof Date){ - return [undefined, this.options.cast.date(value, context)]; - }else if(type === 'object' && value !== null){ - return [undefined, this.options.cast.object(value, context)]; - }else { - return [undefined, value, value]; - } - }catch(err){ - return [err]; - } - } - normalize_columns(columns){ - if(columns === undefined || columns === null){ - return []; - } - if(typeof columns !== 'object'){ - return [Error('Invalid option "columns": expect an array or an object')]; - } - if(!Array.isArray(columns)){ - const newcolumns = []; - for(const k in columns){ - newcolumns.push({ - key: k, - header: columns[k] - }); - } - columns = newcolumns; - }else { - const newcolumns = []; - for(const column of columns){ - if(typeof column === 'string'){ - newcolumns.push({ - key: column, - header: column - }); - }else if(typeof column === 'object' && column !== undefined && !Array.isArray(column)){ - if(!column.key){ - return [Error('Invalid column definition: property "key" is required')]; - } - if(column.header === undefined){ - column.header = column.key; - } - newcolumns.push(column); - }else { - return [Error('Invalid column definition: expect a string or an object')]; - } - } - columns = newcolumns; - } - return [undefined, columns]; - } } const stringify = function(){ @@ -5517,7 +5540,7 @@ const stringify = function(){ const type = typeof argument; if(data === undefined && (Array.isArray(argument))){ data = argument; - }else if(options === undefined && isObject(argument)){ + }else if(options === undefined && is_object(argument)){ options = argument; }else if(callback === undefined && type === 'function'){ callback = argument; diff --git a/packages/csv-stringify/dist/esm/sync.js b/packages/csv-stringify/dist/esm/sync.js index c4168f0e5..9a9b37d4d 100644 --- a/packages/csv-stringify/dist/esm/sync.js +++ b/packages/csv-stringify/dist/esm/sync.js @@ -199,7 +199,7 @@ function write (buffer, value, offset, isLE, mLen, nBytes) { var toString = {}.toString; -var isArray$1 = Array.isArray || function (arr) { +var isArray = Array.isArray || function (arr) { return toString.call(arr) == '[object Array]'; }; @@ -467,7 +467,7 @@ function fromObject (that, obj) { return fromArrayLike(that, obj) } - if (obj.type === 'Buffer' && isArray$1(obj.data)) { + if (obj.type === 'Buffer' && isArray(obj.data)) { return fromArrayLike(that, obj.data) } } @@ -532,7 +532,7 @@ Buffer.isEncoding = function isEncoding (encoding) { }; Buffer.concat = function concat (list, length) { - if (!isArray$1(list)) { + if (!isArray(list)) { throw new TypeError('"list" argument must be an Array of Buffers') } @@ -1963,3038 +1963,6 @@ function isSlowBuffer (obj) { return typeof obj.readFloatLE === 'function' && typeof obj.slice === 'function' && isFastBuffer(obj.slice(0, 0)) } -var domain; - -// This constructor is used to store event handlers. Instantiating this is -// faster than explicitly calling `Object.create(null)` to get a "clean" empty -// object (tested with v8 v4.9). -function EventHandlers() {} -EventHandlers.prototype = Object.create(null); - -function EventEmitter() { - EventEmitter.init.call(this); -} - -// nodejs oddity -// require('events') === require('events').EventEmitter -EventEmitter.EventEmitter = EventEmitter; - -EventEmitter.usingDomains = false; - -EventEmitter.prototype.domain = undefined; -EventEmitter.prototype._events = undefined; -EventEmitter.prototype._maxListeners = undefined; - -// By default EventEmitters will print a warning if more than 10 listeners are -// added to it. This is a useful default which helps finding memory leaks. -EventEmitter.defaultMaxListeners = 10; - -EventEmitter.init = function() { - this.domain = null; - if (EventEmitter.usingDomains) { - // if there is an active domain, then attach to it. - if (domain.active ) ; - } - - if (!this._events || this._events === Object.getPrototypeOf(this)._events) { - this._events = new EventHandlers(); - this._eventsCount = 0; - } - - this._maxListeners = this._maxListeners || undefined; -}; - -// Obviously not all Emitters should be limited to 10. This function allows -// that to be increased. Set to zero for unlimited. -EventEmitter.prototype.setMaxListeners = function setMaxListeners(n) { - if (typeof n !== 'number' || n < 0 || isNaN(n)) - throw new TypeError('"n" argument must be a positive number'); - this._maxListeners = n; - return this; -}; - -function $getMaxListeners(that) { - if (that._maxListeners === undefined) - return EventEmitter.defaultMaxListeners; - return that._maxListeners; -} - -EventEmitter.prototype.getMaxListeners = function getMaxListeners() { - return $getMaxListeners(this); -}; - -// These standalone emit* functions are used to optimize calling of event -// handlers for fast cases because emit() itself often has a variable number of -// arguments and can be deoptimized because of that. These functions always have -// the same number of arguments and thus do not get deoptimized, so the code -// inside them can execute faster. -function emitNone(handler, isFn, self) { - if (isFn) - handler.call(self); - else { - var len = handler.length; - var listeners = arrayClone(handler, len); - for (var i = 0; i < len; ++i) - listeners[i].call(self); - } -} -function emitOne(handler, isFn, self, arg1) { - if (isFn) - handler.call(self, arg1); - else { - var len = handler.length; - var listeners = arrayClone(handler, len); - for (var i = 0; i < len; ++i) - listeners[i].call(self, arg1); - } -} -function emitTwo(handler, isFn, self, arg1, arg2) { - if (isFn) - handler.call(self, arg1, arg2); - else { - var len = handler.length; - var listeners = arrayClone(handler, len); - for (var i = 0; i < len; ++i) - listeners[i].call(self, arg1, arg2); - } -} -function emitThree(handler, isFn, self, arg1, arg2, arg3) { - if (isFn) - handler.call(self, arg1, arg2, arg3); - else { - var len = handler.length; - var listeners = arrayClone(handler, len); - for (var i = 0; i < len; ++i) - listeners[i].call(self, arg1, arg2, arg3); - } -} - -function emitMany(handler, isFn, self, args) { - if (isFn) - handler.apply(self, args); - else { - var len = handler.length; - var listeners = arrayClone(handler, len); - for (var i = 0; i < len; ++i) - listeners[i].apply(self, args); - } -} - -EventEmitter.prototype.emit = function emit(type) { - var er, handler, len, args, i, events, domain; - var doError = (type === 'error'); - - events = this._events; - if (events) - doError = (doError && events.error == null); - else if (!doError) - return false; - - domain = this.domain; - - // If there is no 'error' event listener then throw. - if (doError) { - er = arguments[1]; - if (domain) { - if (!er) - er = new Error('Uncaught, unspecified "error" event'); - er.domainEmitter = this; - er.domain = domain; - er.domainThrown = false; - domain.emit('error', er); - } else if (er instanceof Error) { - throw er; // Unhandled 'error' event - } else { - // At least give some kind of context to the user - var err = new Error('Uncaught, unspecified "error" event. (' + er + ')'); - err.context = er; - throw err; - } - return false; - } - - handler = events[type]; - - if (!handler) - return false; - - var isFn = typeof handler === 'function'; - len = arguments.length; - switch (len) { - // fast cases - case 1: - emitNone(handler, isFn, this); - break; - case 2: - emitOne(handler, isFn, this, arguments[1]); - break; - case 3: - emitTwo(handler, isFn, this, arguments[1], arguments[2]); - break; - case 4: - emitThree(handler, isFn, this, arguments[1], arguments[2], arguments[3]); - break; - // slower - default: - args = new Array(len - 1); - for (i = 1; i < len; i++) - args[i - 1] = arguments[i]; - emitMany(handler, isFn, this, args); - } - - return true; -}; - -function _addListener(target, type, listener, prepend) { - var m; - var events; - var existing; - - if (typeof listener !== 'function') - throw new TypeError('"listener" argument must be a function'); - - events = target._events; - if (!events) { - events = target._events = new EventHandlers(); - target._eventsCount = 0; - } else { - // To avoid recursion in the case that type === "newListener"! Before - // adding it to the listeners, first emit "newListener". - if (events.newListener) { - target.emit('newListener', type, - listener.listener ? listener.listener : listener); - - // Re-assign `events` because a newListener handler could have caused the - // this._events to be assigned to a new object - events = target._events; - } - existing = events[type]; - } - - if (!existing) { - // Optimize the case of one listener. Don't need the extra array object. - existing = events[type] = listener; - ++target._eventsCount; - } else { - if (typeof existing === 'function') { - // Adding the second element, need to change to array. - existing = events[type] = prepend ? [listener, existing] : - [existing, listener]; - } else { - // If we've already got an array, just append. - if (prepend) { - existing.unshift(listener); - } else { - existing.push(listener); - } - } - - // Check for listener leak - if (!existing.warned) { - m = $getMaxListeners(target); - if (m && m > 0 && existing.length > m) { - existing.warned = true; - var w = new Error('Possible EventEmitter memory leak detected. ' + - existing.length + ' ' + type + ' listeners added. ' + - 'Use emitter.setMaxListeners() to increase limit'); - w.name = 'MaxListenersExceededWarning'; - w.emitter = target; - w.type = type; - w.count = existing.length; - emitWarning(w); - } - } - } - - return target; -} -function emitWarning(e) { - typeof console.warn === 'function' ? console.warn(e) : console.log(e); -} -EventEmitter.prototype.addListener = function addListener(type, listener) { - return _addListener(this, type, listener, false); -}; - -EventEmitter.prototype.on = EventEmitter.prototype.addListener; - -EventEmitter.prototype.prependListener = - function prependListener(type, listener) { - return _addListener(this, type, listener, true); - }; - -function _onceWrap(target, type, listener) { - var fired = false; - function g() { - target.removeListener(type, g); - if (!fired) { - fired = true; - listener.apply(target, arguments); - } - } - g.listener = listener; - return g; -} - -EventEmitter.prototype.once = function once(type, listener) { - if (typeof listener !== 'function') - throw new TypeError('"listener" argument must be a function'); - this.on(type, _onceWrap(this, type, listener)); - return this; -}; - -EventEmitter.prototype.prependOnceListener = - function prependOnceListener(type, listener) { - if (typeof listener !== 'function') - throw new TypeError('"listener" argument must be a function'); - this.prependListener(type, _onceWrap(this, type, listener)); - return this; - }; - -// emits a 'removeListener' event iff the listener was removed -EventEmitter.prototype.removeListener = - function removeListener(type, listener) { - var list, events, position, i, originalListener; - - if (typeof listener !== 'function') - throw new TypeError('"listener" argument must be a function'); - - events = this._events; - if (!events) - return this; - - list = events[type]; - if (!list) - return this; - - if (list === listener || (list.listener && list.listener === listener)) { - if (--this._eventsCount === 0) - this._events = new EventHandlers(); - else { - delete events[type]; - if (events.removeListener) - this.emit('removeListener', type, list.listener || listener); - } - } else if (typeof list !== 'function') { - position = -1; - - for (i = list.length; i-- > 0;) { - if (list[i] === listener || - (list[i].listener && list[i].listener === listener)) { - originalListener = list[i].listener; - position = i; - break; - } - } - - if (position < 0) - return this; - - if (list.length === 1) { - list[0] = undefined; - if (--this._eventsCount === 0) { - this._events = new EventHandlers(); - return this; - } else { - delete events[type]; - } - } else { - spliceOne(list, position); - } - - if (events.removeListener) - this.emit('removeListener', type, originalListener || listener); - } - - return this; - }; - -EventEmitter.prototype.removeAllListeners = - function removeAllListeners(type) { - var listeners, events; - - events = this._events; - if (!events) - return this; - - // not listening for removeListener, no need to emit - if (!events.removeListener) { - if (arguments.length === 0) { - this._events = new EventHandlers(); - this._eventsCount = 0; - } else if (events[type]) { - if (--this._eventsCount === 0) - this._events = new EventHandlers(); - else - delete events[type]; - } - return this; - } - - // emit removeListener for all listeners on all events - if (arguments.length === 0) { - var keys = Object.keys(events); - for (var i = 0, key; i < keys.length; ++i) { - key = keys[i]; - if (key === 'removeListener') continue; - this.removeAllListeners(key); - } - this.removeAllListeners('removeListener'); - this._events = new EventHandlers(); - this._eventsCount = 0; - return this; - } - - listeners = events[type]; - - if (typeof listeners === 'function') { - this.removeListener(type, listeners); - } else if (listeners) { - // LIFO order - do { - this.removeListener(type, listeners[listeners.length - 1]); - } while (listeners[0]); - } - - return this; - }; - -EventEmitter.prototype.listeners = function listeners(type) { - var evlistener; - var ret; - var events = this._events; - - if (!events) - ret = []; - else { - evlistener = events[type]; - if (!evlistener) - ret = []; - else if (typeof evlistener === 'function') - ret = [evlistener.listener || evlistener]; - else - ret = unwrapListeners(evlistener); - } - - return ret; -}; - -EventEmitter.listenerCount = function(emitter, type) { - if (typeof emitter.listenerCount === 'function') { - return emitter.listenerCount(type); - } else { - return listenerCount$1.call(emitter, type); - } -}; - -EventEmitter.prototype.listenerCount = listenerCount$1; -function listenerCount$1(type) { - var events = this._events; - - if (events) { - var evlistener = events[type]; - - if (typeof evlistener === 'function') { - return 1; - } else if (evlistener) { - return evlistener.length; - } - } - - return 0; -} - -EventEmitter.prototype.eventNames = function eventNames() { - return this._eventsCount > 0 ? Reflect.ownKeys(this._events) : []; -}; - -// About 1.5x faster than the two-arg version of Array#splice(). -function spliceOne(list, index) { - for (var i = index, k = i + 1, n = list.length; k < n; i += 1, k += 1) - list[i] = list[k]; - list.pop(); -} - -function arrayClone(arr, i) { - var copy = new Array(i); - while (i--) - copy[i] = arr[i]; - return copy; -} - -function unwrapListeners(arr) { - var ret = new Array(arr.length); - for (var i = 0; i < ret.length; ++i) { - ret[i] = arr[i].listener || arr[i]; - } - return ret; -} - -// shim for using process in browser -// based off https://github.com/defunctzombie/node-process/blob/master/browser.js - -function defaultSetTimout() { - throw new Error('setTimeout has not been defined'); -} -function defaultClearTimeout () { - throw new Error('clearTimeout has not been defined'); -} -var cachedSetTimeout = defaultSetTimout; -var cachedClearTimeout = defaultClearTimeout; -if (typeof global$1.setTimeout === 'function') { - cachedSetTimeout = setTimeout; -} -if (typeof global$1.clearTimeout === 'function') { - cachedClearTimeout = clearTimeout; -} - -function runTimeout(fun) { - if (cachedSetTimeout === setTimeout) { - //normal enviroments in sane situations - return setTimeout(fun, 0); - } - // if setTimeout wasn't available but was latter defined - if ((cachedSetTimeout === defaultSetTimout || !cachedSetTimeout) && setTimeout) { - cachedSetTimeout = setTimeout; - return setTimeout(fun, 0); - } - try { - // when when somebody has screwed with setTimeout but no I.E. maddness - return cachedSetTimeout(fun, 0); - } catch(e){ - try { - // When we are in I.E. but the script has been evaled so I.E. doesn't trust the global object when called normally - return cachedSetTimeout.call(null, fun, 0); - } catch(e){ - // same as above but when it's a version of I.E. that must have the global object for 'this', hopfully our context correct otherwise it will throw a global error - return cachedSetTimeout.call(this, fun, 0); - } - } - - -} -function runClearTimeout(marker) { - if (cachedClearTimeout === clearTimeout) { - //normal enviroments in sane situations - return clearTimeout(marker); - } - // if clearTimeout wasn't available but was latter defined - if ((cachedClearTimeout === defaultClearTimeout || !cachedClearTimeout) && clearTimeout) { - cachedClearTimeout = clearTimeout; - return clearTimeout(marker); - } - try { - // when when somebody has screwed with setTimeout but no I.E. maddness - return cachedClearTimeout(marker); - } catch (e){ - try { - // When we are in I.E. but the script has been evaled so I.E. doesn't trust the global object when called normally - return cachedClearTimeout.call(null, marker); - } catch (e){ - // same as above but when it's a version of I.E. that must have the global object for 'this', hopfully our context correct otherwise it will throw a global error. - // Some versions of I.E. have different rules for clearTimeout vs setTimeout - return cachedClearTimeout.call(this, marker); - } - } - - - -} -var queue = []; -var draining = false; -var currentQueue; -var queueIndex = -1; - -function cleanUpNextTick() { - if (!draining || !currentQueue) { - return; - } - draining = false; - if (currentQueue.length) { - queue = currentQueue.concat(queue); - } else { - queueIndex = -1; - } - if (queue.length) { - drainQueue(); - } -} - -function drainQueue() { - if (draining) { - return; - } - var timeout = runTimeout(cleanUpNextTick); - draining = true; - - var len = queue.length; - while(len) { - currentQueue = queue; - queue = []; - while (++queueIndex < len) { - if (currentQueue) { - currentQueue[queueIndex].run(); - } - } - queueIndex = -1; - len = queue.length; - } - currentQueue = null; - draining = false; - runClearTimeout(timeout); -} -function nextTick(fun) { - var args = new Array(arguments.length - 1); - if (arguments.length > 1) { - for (var i = 1; i < arguments.length; i++) { - args[i - 1] = arguments[i]; - } - } - queue.push(new Item(fun, args)); - if (queue.length === 1 && !draining) { - runTimeout(drainQueue); - } -} -// v8 likes predictible objects -function Item(fun, array) { - this.fun = fun; - this.array = array; -} -Item.prototype.run = function () { - this.fun.apply(null, this.array); -}; - -// from https://github.com/kumavis/browser-process-hrtime/blob/master/index.js -var performance = global$1.performance || {}; -performance.now || - performance.mozNow || - performance.msNow || - performance.oNow || - performance.webkitNow || - function(){ return (new Date()).getTime() }; - -var inherits; -if (typeof Object.create === 'function'){ - inherits = function inherits(ctor, superCtor) { - // implementation from standard node.js 'util' module - ctor.super_ = superCtor; - ctor.prototype = Object.create(superCtor.prototype, { - constructor: { - value: ctor, - enumerable: false, - writable: true, - configurable: true - } - }); - }; -} else { - inherits = function inherits(ctor, superCtor) { - ctor.super_ = superCtor; - var TempCtor = function () {}; - TempCtor.prototype = superCtor.prototype; - ctor.prototype = new TempCtor(); - ctor.prototype.constructor = ctor; - }; -} -var inherits$1 = inherits; - -var formatRegExp = /%[sdj%]/g; -function format(f) { - if (!isString(f)) { - var objects = []; - for (var i = 0; i < arguments.length; i++) { - objects.push(inspect(arguments[i])); - } - return objects.join(' '); - } - - var i = 1; - var args = arguments; - var len = args.length; - var str = String(f).replace(formatRegExp, function(x) { - if (x === '%%') return '%'; - if (i >= len) return x; - switch (x) { - case '%s': return String(args[i++]); - case '%d': return Number(args[i++]); - case '%j': - try { - return JSON.stringify(args[i++]); - } catch (_) { - return '[Circular]'; - } - default: - return x; - } - }); - for (var x = args[i]; i < len; x = args[++i]) { - if (isNull(x) || !isObject$1(x)) { - str += ' ' + x; - } else { - str += ' ' + inspect(x); - } - } - return str; -} - -// Mark that a method should not be used. -// Returns a modified function which warns once by default. -// If --no-deprecation is set, then it is a no-op. -function deprecate(fn, msg) { - // Allow for deprecating things in the process of starting up. - if (isUndefined(global$1.process)) { - return function() { - return deprecate(fn, msg).apply(this, arguments); - }; - } - - var warned = false; - function deprecated() { - if (!warned) { - { - console.error(msg); - } - warned = true; - } - return fn.apply(this, arguments); - } - - return deprecated; -} - -var debugs = {}; -var debugEnviron; -function debuglog(set) { - if (isUndefined(debugEnviron)) - debugEnviron = ''; - set = set.toUpperCase(); - if (!debugs[set]) { - if (new RegExp('\\b' + set + '\\b', 'i').test(debugEnviron)) { - var pid = 0; - debugs[set] = function() { - var msg = format.apply(null, arguments); - console.error('%s %d: %s', set, pid, msg); - }; - } else { - debugs[set] = function() {}; - } - } - return debugs[set]; -} - -/** - * Echos the value of a value. Trys to print the value out - * in the best way possible given the different types. - * - * @param {Object} obj The object to print out. - * @param {Object} opts Optional options object that alters the output. - */ -/* legacy: obj, showHidden, depth, colors*/ -function inspect(obj, opts) { - // default options - var ctx = { - seen: [], - stylize: stylizeNoColor - }; - // legacy... - if (arguments.length >= 3) ctx.depth = arguments[2]; - if (arguments.length >= 4) ctx.colors = arguments[3]; - if (isBoolean(opts)) { - // legacy... - ctx.showHidden = opts; - } else if (opts) { - // got an "options" object - _extend(ctx, opts); - } - // set default options - if (isUndefined(ctx.showHidden)) ctx.showHidden = false; - if (isUndefined(ctx.depth)) ctx.depth = 2; - if (isUndefined(ctx.colors)) ctx.colors = false; - if (isUndefined(ctx.customInspect)) ctx.customInspect = true; - if (ctx.colors) ctx.stylize = stylizeWithColor; - return formatValue(ctx, obj, ctx.depth); -} - -// http://en.wikipedia.org/wiki/ANSI_escape_code#graphics -inspect.colors = { - 'bold' : [1, 22], - 'italic' : [3, 23], - 'underline' : [4, 24], - 'inverse' : [7, 27], - 'white' : [37, 39], - 'grey' : [90, 39], - 'black' : [30, 39], - 'blue' : [34, 39], - 'cyan' : [36, 39], - 'green' : [32, 39], - 'magenta' : [35, 39], - 'red' : [31, 39], - 'yellow' : [33, 39] -}; - -// Don't use 'blue' not visible on cmd.exe -inspect.styles = { - 'special': 'cyan', - 'number': 'yellow', - 'boolean': 'yellow', - 'undefined': 'grey', - 'null': 'bold', - 'string': 'green', - 'date': 'magenta', - // "name": intentionally not styling - 'regexp': 'red' -}; - - -function stylizeWithColor(str, styleType) { - var style = inspect.styles[styleType]; - - if (style) { - return '\u001b[' + inspect.colors[style][0] + 'm' + str + - '\u001b[' + inspect.colors[style][1] + 'm'; - } else { - return str; - } -} - - -function stylizeNoColor(str, styleType) { - return str; -} - - -function arrayToHash(array) { - var hash = {}; - - array.forEach(function(val, idx) { - hash[val] = true; - }); - - return hash; -} - - -function formatValue(ctx, value, recurseTimes) { - // Provide a hook for user-specified inspect functions. - // Check that value is an object with an inspect function on it - if (ctx.customInspect && - value && - isFunction(value.inspect) && - // Filter out the util module, it's inspect function is special - value.inspect !== inspect && - // Also filter out any prototype objects using the circular check. - !(value.constructor && value.constructor.prototype === value)) { - var ret = value.inspect(recurseTimes, ctx); - if (!isString(ret)) { - ret = formatValue(ctx, ret, recurseTimes); - } - return ret; - } - - // Primitive types cannot have properties - var primitive = formatPrimitive(ctx, value); - if (primitive) { - return primitive; - } - - // Look up the keys of the object. - var keys = Object.keys(value); - var visibleKeys = arrayToHash(keys); - - if (ctx.showHidden) { - keys = Object.getOwnPropertyNames(value); - } - - // IE doesn't make error fields non-enumerable - // http://msdn.microsoft.com/en-us/library/ie/dww52sbt(v=vs.94).aspx - if (isError(value) - && (keys.indexOf('message') >= 0 || keys.indexOf('description') >= 0)) { - return formatError(value); - } - - // Some type of object without properties can be shortcutted. - if (keys.length === 0) { - if (isFunction(value)) { - var name = value.name ? ': ' + value.name : ''; - return ctx.stylize('[Function' + name + ']', 'special'); - } - if (isRegExp(value)) { - return ctx.stylize(RegExp.prototype.toString.call(value), 'regexp'); - } - if (isDate(value)) { - return ctx.stylize(Date.prototype.toString.call(value), 'date'); - } - if (isError(value)) { - return formatError(value); - } - } - - var base = '', array = false, braces = ['{', '}']; - - // Make Array say that they are Array - if (isArray(value)) { - array = true; - braces = ['[', ']']; - } - - // Make functions say that they are functions - if (isFunction(value)) { - var n = value.name ? ': ' + value.name : ''; - base = ' [Function' + n + ']'; - } - - // Make RegExps say that they are RegExps - if (isRegExp(value)) { - base = ' ' + RegExp.prototype.toString.call(value); - } - - // Make dates with properties first say the date - if (isDate(value)) { - base = ' ' + Date.prototype.toUTCString.call(value); - } - - // Make error with message first say the error - if (isError(value)) { - base = ' ' + formatError(value); - } - - if (keys.length === 0 && (!array || value.length == 0)) { - return braces[0] + base + braces[1]; - } - - if (recurseTimes < 0) { - if (isRegExp(value)) { - return ctx.stylize(RegExp.prototype.toString.call(value), 'regexp'); - } else { - return ctx.stylize('[Object]', 'special'); - } - } - - ctx.seen.push(value); - - var output; - if (array) { - output = formatArray(ctx, value, recurseTimes, visibleKeys, keys); - } else { - output = keys.map(function(key) { - return formatProperty(ctx, value, recurseTimes, visibleKeys, key, array); - }); - } - - ctx.seen.pop(); - - return reduceToSingleString(output, base, braces); -} - - -function formatPrimitive(ctx, value) { - if (isUndefined(value)) - return ctx.stylize('undefined', 'undefined'); - if (isString(value)) { - var simple = '\'' + JSON.stringify(value).replace(/^"|"$/g, '') - .replace(/'/g, "\\'") - .replace(/\\"/g, '"') + '\''; - return ctx.stylize(simple, 'string'); - } - if (isNumber(value)) - return ctx.stylize('' + value, 'number'); - if (isBoolean(value)) - return ctx.stylize('' + value, 'boolean'); - // For some reason typeof null is "object", so special case here. - if (isNull(value)) - return ctx.stylize('null', 'null'); -} - - -function formatError(value) { - return '[' + Error.prototype.toString.call(value) + ']'; -} - - -function formatArray(ctx, value, recurseTimes, visibleKeys, keys) { - var output = []; - for (var i = 0, l = value.length; i < l; ++i) { - if (hasOwnProperty(value, String(i))) { - output.push(formatProperty(ctx, value, recurseTimes, visibleKeys, - String(i), true)); - } else { - output.push(''); - } - } - keys.forEach(function(key) { - if (!key.match(/^\d+$/)) { - output.push(formatProperty(ctx, value, recurseTimes, visibleKeys, - key, true)); - } - }); - return output; -} - - -function formatProperty(ctx, value, recurseTimes, visibleKeys, key, array) { - var name, str, desc; - desc = Object.getOwnPropertyDescriptor(value, key) || { value: value[key] }; - if (desc.get) { - if (desc.set) { - str = ctx.stylize('[Getter/Setter]', 'special'); - } else { - str = ctx.stylize('[Getter]', 'special'); - } - } else { - if (desc.set) { - str = ctx.stylize('[Setter]', 'special'); - } - } - if (!hasOwnProperty(visibleKeys, key)) { - name = '[' + key + ']'; - } - if (!str) { - if (ctx.seen.indexOf(desc.value) < 0) { - if (isNull(recurseTimes)) { - str = formatValue(ctx, desc.value, null); - } else { - str = formatValue(ctx, desc.value, recurseTimes - 1); - } - if (str.indexOf('\n') > -1) { - if (array) { - str = str.split('\n').map(function(line) { - return ' ' + line; - }).join('\n').substr(2); - } else { - str = '\n' + str.split('\n').map(function(line) { - return ' ' + line; - }).join('\n'); - } - } - } else { - str = ctx.stylize('[Circular]', 'special'); - } - } - if (isUndefined(name)) { - if (array && key.match(/^\d+$/)) { - return str; - } - name = JSON.stringify('' + key); - if (name.match(/^"([a-zA-Z_][a-zA-Z_0-9]*)"$/)) { - name = name.substr(1, name.length - 2); - name = ctx.stylize(name, 'name'); - } else { - name = name.replace(/'/g, "\\'") - .replace(/\\"/g, '"') - .replace(/(^"|"$)/g, "'"); - name = ctx.stylize(name, 'string'); - } - } - - return name + ': ' + str; -} - - -function reduceToSingleString(output, base, braces) { - var length = output.reduce(function(prev, cur) { - if (cur.indexOf('\n') >= 0) ; - return prev + cur.replace(/\u001b\[\d\d?m/g, '').length + 1; - }, 0); - - if (length > 60) { - return braces[0] + - (base === '' ? '' : base + '\n ') + - ' ' + - output.join(',\n ') + - ' ' + - braces[1]; - } - - return braces[0] + base + ' ' + output.join(', ') + ' ' + braces[1]; -} - - -// NOTE: These type checking functions intentionally don't use `instanceof` -// because it is fragile and can be easily faked with `Object.create()`. -function isArray(ar) { - return Array.isArray(ar); -} - -function isBoolean(arg) { - return typeof arg === 'boolean'; -} - -function isNull(arg) { - return arg === null; -} - -function isNumber(arg) { - return typeof arg === 'number'; -} - -function isString(arg) { - return typeof arg === 'string'; -} - -function isUndefined(arg) { - return arg === void 0; -} - -function isRegExp(re) { - return isObject$1(re) && objectToString(re) === '[object RegExp]'; -} - -function isObject$1(arg) { - return typeof arg === 'object' && arg !== null; -} - -function isDate(d) { - return isObject$1(d) && objectToString(d) === '[object Date]'; -} - -function isError(e) { - return isObject$1(e) && - (objectToString(e) === '[object Error]' || e instanceof Error); -} - -function isFunction(arg) { - return typeof arg === 'function'; -} - -function objectToString(o) { - return Object.prototype.toString.call(o); -} - -function _extend(origin, add) { - // Don't do anything if add isn't an object - if (!add || !isObject$1(add)) return origin; - - var keys = Object.keys(add); - var i = keys.length; - while (i--) { - origin[keys[i]] = add[keys[i]]; - } - return origin; -} -function hasOwnProperty(obj, prop) { - return Object.prototype.hasOwnProperty.call(obj, prop); -} - -function BufferList() { - this.head = null; - this.tail = null; - this.length = 0; -} - -BufferList.prototype.push = function (v) { - var entry = { data: v, next: null }; - if (this.length > 0) this.tail.next = entry;else this.head = entry; - this.tail = entry; - ++this.length; -}; - -BufferList.prototype.unshift = function (v) { - var entry = { data: v, next: this.head }; - if (this.length === 0) this.tail = entry; - this.head = entry; - ++this.length; -}; - -BufferList.prototype.shift = function () { - if (this.length === 0) return; - var ret = this.head.data; - if (this.length === 1) this.head = this.tail = null;else this.head = this.head.next; - --this.length; - return ret; -}; - -BufferList.prototype.clear = function () { - this.head = this.tail = null; - this.length = 0; -}; - -BufferList.prototype.join = function (s) { - if (this.length === 0) return ''; - var p = this.head; - var ret = '' + p.data; - while (p = p.next) { - ret += s + p.data; - }return ret; -}; - -BufferList.prototype.concat = function (n) { - if (this.length === 0) return Buffer.alloc(0); - if (this.length === 1) return this.head.data; - var ret = Buffer.allocUnsafe(n >>> 0); - var p = this.head; - var i = 0; - while (p) { - p.data.copy(ret, i); - i += p.data.length; - p = p.next; - } - return ret; -}; - -// Copyright Joyent, Inc. and other Node contributors. -var isBufferEncoding = Buffer.isEncoding - || function(encoding) { - switch (encoding && encoding.toLowerCase()) { - case 'hex': case 'utf8': case 'utf-8': case 'ascii': case 'binary': case 'base64': case 'ucs2': case 'ucs-2': case 'utf16le': case 'utf-16le': case 'raw': return true; - default: return false; - } - }; - - -function assertEncoding(encoding) { - if (encoding && !isBufferEncoding(encoding)) { - throw new Error('Unknown encoding: ' + encoding); - } -} - -// StringDecoder provides an interface for efficiently splitting a series of -// buffers into a series of JS strings without breaking apart multi-byte -// characters. CESU-8 is handled as part of the UTF-8 encoding. -// -// @TODO Handling all encodings inside a single object makes it very difficult -// to reason about this code, so it should be split up in the future. -// @TODO There should be a utf8-strict encoding that rejects invalid UTF-8 code -// points as used by CESU-8. -function StringDecoder(encoding) { - this.encoding = (encoding || 'utf8').toLowerCase().replace(/[-_]/, ''); - assertEncoding(encoding); - switch (this.encoding) { - case 'utf8': - // CESU-8 represents each of Surrogate Pair by 3-bytes - this.surrogateSize = 3; - break; - case 'ucs2': - case 'utf16le': - // UTF-16 represents each of Surrogate Pair by 2-bytes - this.surrogateSize = 2; - this.detectIncompleteChar = utf16DetectIncompleteChar; - break; - case 'base64': - // Base-64 stores 3 bytes in 4 chars, and pads the remainder. - this.surrogateSize = 3; - this.detectIncompleteChar = base64DetectIncompleteChar; - break; - default: - this.write = passThroughWrite; - return; - } - - // Enough space to store all bytes of a single character. UTF-8 needs 4 - // bytes, but CESU-8 may require up to 6 (3 bytes per surrogate). - this.charBuffer = new Buffer(6); - // Number of bytes received for the current incomplete multi-byte character. - this.charReceived = 0; - // Number of bytes expected for the current incomplete multi-byte character. - this.charLength = 0; -} - -// write decodes the given buffer and returns it as JS string that is -// guaranteed to not contain any partial multi-byte characters. Any partial -// character found at the end of the buffer is buffered up, and will be -// returned when calling write again with the remaining bytes. -// -// Note: Converting a Buffer containing an orphan surrogate to a String -// currently works, but converting a String to a Buffer (via `new Buffer`, or -// Buffer#write) will replace incomplete surrogates with the unicode -// replacement character. See https://codereview.chromium.org/121173009/ . -StringDecoder.prototype.write = function(buffer) { - var charStr = ''; - // if our last write ended with an incomplete multibyte character - while (this.charLength) { - // determine how many remaining bytes this buffer has to offer for this char - var available = (buffer.length >= this.charLength - this.charReceived) ? - this.charLength - this.charReceived : - buffer.length; - - // add the new bytes to the char buffer - buffer.copy(this.charBuffer, this.charReceived, 0, available); - this.charReceived += available; - - if (this.charReceived < this.charLength) { - // still not enough chars in this buffer? wait for more ... - return ''; - } - - // remove bytes belonging to the current character from the buffer - buffer = buffer.slice(available, buffer.length); - - // get the character that was split - charStr = this.charBuffer.slice(0, this.charLength).toString(this.encoding); - - // CESU-8: lead surrogate (D800-DBFF) is also the incomplete character - var charCode = charStr.charCodeAt(charStr.length - 1); - if (charCode >= 0xD800 && charCode <= 0xDBFF) { - this.charLength += this.surrogateSize; - charStr = ''; - continue; - } - this.charReceived = this.charLength = 0; - - // if there are no more bytes in this buffer, just emit our char - if (buffer.length === 0) { - return charStr; - } - break; - } - - // determine and set charLength / charReceived - this.detectIncompleteChar(buffer); - - var end = buffer.length; - if (this.charLength) { - // buffer the incomplete character bytes we got - buffer.copy(this.charBuffer, 0, buffer.length - this.charReceived, end); - end -= this.charReceived; - } - - charStr += buffer.toString(this.encoding, 0, end); - - var end = charStr.length - 1; - var charCode = charStr.charCodeAt(end); - // CESU-8: lead surrogate (D800-DBFF) is also the incomplete character - if (charCode >= 0xD800 && charCode <= 0xDBFF) { - var size = this.surrogateSize; - this.charLength += size; - this.charReceived += size; - this.charBuffer.copy(this.charBuffer, size, 0, size); - buffer.copy(this.charBuffer, 0, 0, size); - return charStr.substring(0, end); - } - - // or just emit the charStr - return charStr; -}; - -// detectIncompleteChar determines if there is an incomplete UTF-8 character at -// the end of the given buffer. If so, it sets this.charLength to the byte -// length that character, and sets this.charReceived to the number of bytes -// that are available for this character. -StringDecoder.prototype.detectIncompleteChar = function(buffer) { - // determine how many bytes we have to check at the end of this buffer - var i = (buffer.length >= 3) ? 3 : buffer.length; - - // Figure out if one of the last i bytes of our buffer announces an - // incomplete char. - for (; i > 0; i--) { - var c = buffer[buffer.length - i]; - - // See http://en.wikipedia.org/wiki/UTF-8#Description - - // 110XXXXX - if (i == 1 && c >> 5 == 0x06) { - this.charLength = 2; - break; - } - - // 1110XXXX - if (i <= 2 && c >> 4 == 0x0E) { - this.charLength = 3; - break; - } - - // 11110XXX - if (i <= 3 && c >> 3 == 0x1E) { - this.charLength = 4; - break; - } - } - this.charReceived = i; -}; - -StringDecoder.prototype.end = function(buffer) { - var res = ''; - if (buffer && buffer.length) - res = this.write(buffer); - - if (this.charReceived) { - var cr = this.charReceived; - var buf = this.charBuffer; - var enc = this.encoding; - res += buf.slice(0, cr).toString(enc); - } - - return res; -}; - -function passThroughWrite(buffer) { - return buffer.toString(this.encoding); -} - -function utf16DetectIncompleteChar(buffer) { - this.charReceived = buffer.length % 2; - this.charLength = this.charReceived ? 2 : 0; -} - -function base64DetectIncompleteChar(buffer) { - this.charReceived = buffer.length % 3; - this.charLength = this.charReceived ? 3 : 0; -} - -Readable.ReadableState = ReadableState; - -var debug = debuglog('stream'); -inherits$1(Readable, EventEmitter); - -function prependListener(emitter, event, fn) { - // Sadly this is not cacheable as some libraries bundle their own - // event emitter implementation with them. - if (typeof emitter.prependListener === 'function') { - return emitter.prependListener(event, fn); - } else { - // This is a hack to make sure that our error handler is attached before any - // userland ones. NEVER DO THIS. This is here only because this code needs - // to continue to work with older versions of Node.js that do not include - // the prependListener() method. The goal is to eventually remove this hack. - if (!emitter._events || !emitter._events[event]) - emitter.on(event, fn); - else if (Array.isArray(emitter._events[event])) - emitter._events[event].unshift(fn); - else - emitter._events[event] = [fn, emitter._events[event]]; - } -} -function listenerCount (emitter, type) { - return emitter.listeners(type).length; -} -function ReadableState(options, stream) { - - options = options || {}; - - // object stream flag. Used to make read(n) ignore n and to - // make all the buffer merging and length checks go away - this.objectMode = !!options.objectMode; - - if (stream instanceof Duplex) this.objectMode = this.objectMode || !!options.readableObjectMode; - - // the point at which it stops calling _read() to fill the buffer - // Note: 0 is a valid value, means "don't call _read preemptively ever" - var hwm = options.highWaterMark; - var defaultHwm = this.objectMode ? 16 : 16 * 1024; - this.highWaterMark = hwm || hwm === 0 ? hwm : defaultHwm; - - // cast to ints. - this.highWaterMark = ~ ~this.highWaterMark; - - // A linked list is used to store data chunks instead of an array because the - // linked list can remove elements from the beginning faster than - // array.shift() - this.buffer = new BufferList(); - this.length = 0; - this.pipes = null; - this.pipesCount = 0; - this.flowing = null; - this.ended = false; - this.endEmitted = false; - this.reading = false; - - // a flag to be able to tell if the onwrite cb is called immediately, - // or on a later tick. We set this to true at first, because any - // actions that shouldn't happen until "later" should generally also - // not happen before the first write call. - this.sync = true; - - // whenever we return null, then we set a flag to say - // that we're awaiting a 'readable' event emission. - this.needReadable = false; - this.emittedReadable = false; - this.readableListening = false; - this.resumeScheduled = false; - - // Crypto is kind of old and crusty. Historically, its default string - // encoding is 'binary' so we have to make this configurable. - // Everything else in the universe uses 'utf8', though. - this.defaultEncoding = options.defaultEncoding || 'utf8'; - - // when piping, we only care about 'readable' events that happen - // after read()ing all the bytes and not getting any pushback. - this.ranOut = false; - - // the number of writers that are awaiting a drain event in .pipe()s - this.awaitDrain = 0; - - // if true, a maybeReadMore has been scheduled - this.readingMore = false; - - this.decoder = null; - this.encoding = null; - if (options.encoding) { - this.decoder = new StringDecoder(options.encoding); - this.encoding = options.encoding; - } -} -function Readable(options) { - - if (!(this instanceof Readable)) return new Readable(options); - - this._readableState = new ReadableState(options, this); - - // legacy - this.readable = true; - - if (options && typeof options.read === 'function') this._read = options.read; - - EventEmitter.call(this); -} - -// Manually shove something into the read() buffer. -// This returns true if the highWaterMark has not been hit yet, -// similar to how Writable.write() returns true if you should -// write() some more. -Readable.prototype.push = function (chunk, encoding) { - var state = this._readableState; - - if (!state.objectMode && typeof chunk === 'string') { - encoding = encoding || state.defaultEncoding; - if (encoding !== state.encoding) { - chunk = Buffer.from(chunk, encoding); - encoding = ''; - } - } - - return readableAddChunk(this, state, chunk, encoding, false); -}; - -// Unshift should *always* be something directly out of read() -Readable.prototype.unshift = function (chunk) { - var state = this._readableState; - return readableAddChunk(this, state, chunk, '', true); -}; - -Readable.prototype.isPaused = function () { - return this._readableState.flowing === false; -}; - -function readableAddChunk(stream, state, chunk, encoding, addToFront) { - var er = chunkInvalid(state, chunk); - if (er) { - stream.emit('error', er); - } else if (chunk === null) { - state.reading = false; - onEofChunk(stream, state); - } else if (state.objectMode || chunk && chunk.length > 0) { - if (state.ended && !addToFront) { - var e = new Error('stream.push() after EOF'); - stream.emit('error', e); - } else if (state.endEmitted && addToFront) { - var _e = new Error('stream.unshift() after end event'); - stream.emit('error', _e); - } else { - var skipAdd; - if (state.decoder && !addToFront && !encoding) { - chunk = state.decoder.write(chunk); - skipAdd = !state.objectMode && chunk.length === 0; - } - - if (!addToFront) state.reading = false; - - // Don't add to the buffer if we've decoded to an empty string chunk and - // we're not in object mode - if (!skipAdd) { - // if we want the data now, just emit it. - if (state.flowing && state.length === 0 && !state.sync) { - stream.emit('data', chunk); - stream.read(0); - } else { - // update the buffer info. - state.length += state.objectMode ? 1 : chunk.length; - if (addToFront) state.buffer.unshift(chunk);else state.buffer.push(chunk); - - if (state.needReadable) emitReadable(stream); - } - } - - maybeReadMore(stream, state); - } - } else if (!addToFront) { - state.reading = false; - } - - return needMoreData(state); -} - -// if it's past the high water mark, we can push in some more. -// Also, if we have no data yet, we can stand some -// more bytes. This is to work around cases where hwm=0, -// such as the repl. Also, if the push() triggered a -// readable event, and the user called read(largeNumber) such that -// needReadable was set, then we ought to push more, so that another -// 'readable' event will be triggered. -function needMoreData(state) { - return !state.ended && (state.needReadable || state.length < state.highWaterMark || state.length === 0); -} - -// backwards compatibility. -Readable.prototype.setEncoding = function (enc) { - this._readableState.decoder = new StringDecoder(enc); - this._readableState.encoding = enc; - return this; -}; - -// Don't raise the hwm > 8MB -var MAX_HWM = 0x800000; -function computeNewHighWaterMark(n) { - if (n >= MAX_HWM) { - n = MAX_HWM; - } else { - // Get the next highest power of 2 to prevent increasing hwm excessively in - // tiny amounts - n--; - n |= n >>> 1; - n |= n >>> 2; - n |= n >>> 4; - n |= n >>> 8; - n |= n >>> 16; - n++; - } - return n; -} - -// This function is designed to be inlinable, so please take care when making -// changes to the function body. -function howMuchToRead(n, state) { - if (n <= 0 || state.length === 0 && state.ended) return 0; - if (state.objectMode) return 1; - if (n !== n) { - // Only flow one buffer at a time - if (state.flowing && state.length) return state.buffer.head.data.length;else return state.length; - } - // If we're asking for more than the current hwm, then raise the hwm. - if (n > state.highWaterMark) state.highWaterMark = computeNewHighWaterMark(n); - if (n <= state.length) return n; - // Don't have enough - if (!state.ended) { - state.needReadable = true; - return 0; - } - return state.length; -} - -// you can override either this method, or the async _read(n) below. -Readable.prototype.read = function (n) { - debug('read', n); - n = parseInt(n, 10); - var state = this._readableState; - var nOrig = n; - - if (n !== 0) state.emittedReadable = false; - - // if we're doing read(0) to trigger a readable event, but we - // already have a bunch of data in the buffer, then just trigger - // the 'readable' event and move on. - if (n === 0 && state.needReadable && (state.length >= state.highWaterMark || state.ended)) { - debug('read: emitReadable', state.length, state.ended); - if (state.length === 0 && state.ended) endReadable(this);else emitReadable(this); - return null; - } - - n = howMuchToRead(n, state); - - // if we've ended, and we're now clear, then finish it up. - if (n === 0 && state.ended) { - if (state.length === 0) endReadable(this); - return null; - } - - // All the actual chunk generation logic needs to be - // *below* the call to _read. The reason is that in certain - // synthetic stream cases, such as passthrough streams, _read - // may be a completely synchronous operation which may change - // the state of the read buffer, providing enough data when - // before there was *not* enough. - // - // So, the steps are: - // 1. Figure out what the state of things will be after we do - // a read from the buffer. - // - // 2. If that resulting state will trigger a _read, then call _read. - // Note that this may be asynchronous, or synchronous. Yes, it is - // deeply ugly to write APIs this way, but that still doesn't mean - // that the Readable class should behave improperly, as streams are - // designed to be sync/async agnostic. - // Take note if the _read call is sync or async (ie, if the read call - // has returned yet), so that we know whether or not it's safe to emit - // 'readable' etc. - // - // 3. Actually pull the requested chunks out of the buffer and return. - - // if we need a readable event, then we need to do some reading. - var doRead = state.needReadable; - debug('need readable', doRead); - - // if we currently have less than the highWaterMark, then also read some - if (state.length === 0 || state.length - n < state.highWaterMark) { - doRead = true; - debug('length less than watermark', doRead); - } - - // however, if we've ended, then there's no point, and if we're already - // reading, then it's unnecessary. - if (state.ended || state.reading) { - doRead = false; - debug('reading or ended', doRead); - } else if (doRead) { - debug('do read'); - state.reading = true; - state.sync = true; - // if the length is currently zero, then we *need* a readable event. - if (state.length === 0) state.needReadable = true; - // call internal read method - this._read(state.highWaterMark); - state.sync = false; - // If _read pushed data synchronously, then `reading` will be false, - // and we need to re-evaluate how much data we can return to the user. - if (!state.reading) n = howMuchToRead(nOrig, state); - } - - var ret; - if (n > 0) ret = fromList(n, state);else ret = null; - - if (ret === null) { - state.needReadable = true; - n = 0; - } else { - state.length -= n; - } - - if (state.length === 0) { - // If we have nothing in the buffer, then we want to know - // as soon as we *do* get something into the buffer. - if (!state.ended) state.needReadable = true; - - // If we tried to read() past the EOF, then emit end on the next tick. - if (nOrig !== n && state.ended) endReadable(this); - } - - if (ret !== null) this.emit('data', ret); - - return ret; -}; - -function chunkInvalid(state, chunk) { - var er = null; - if (!isBuffer(chunk) && typeof chunk !== 'string' && chunk !== null && chunk !== undefined && !state.objectMode) { - er = new TypeError('Invalid non-string/buffer chunk'); - } - return er; -} - -function onEofChunk(stream, state) { - if (state.ended) return; - if (state.decoder) { - var chunk = state.decoder.end(); - if (chunk && chunk.length) { - state.buffer.push(chunk); - state.length += state.objectMode ? 1 : chunk.length; - } - } - state.ended = true; - - // emit 'readable' now to make sure it gets picked up. - emitReadable(stream); -} - -// Don't emit readable right away in sync mode, because this can trigger -// another read() call => stack overflow. This way, it might trigger -// a nextTick recursion warning, but that's not so bad. -function emitReadable(stream) { - var state = stream._readableState; - state.needReadable = false; - if (!state.emittedReadable) { - debug('emitReadable', state.flowing); - state.emittedReadable = true; - if (state.sync) nextTick(emitReadable_, stream);else emitReadable_(stream); - } -} - -function emitReadable_(stream) { - debug('emit readable'); - stream.emit('readable'); - flow(stream); -} - -// at this point, the user has presumably seen the 'readable' event, -// and called read() to consume some data. that may have triggered -// in turn another _read(n) call, in which case reading = true if -// it's in progress. -// However, if we're not ended, or reading, and the length < hwm, -// then go ahead and try to read some more preemptively. -function maybeReadMore(stream, state) { - if (!state.readingMore) { - state.readingMore = true; - nextTick(maybeReadMore_, stream, state); - } -} - -function maybeReadMore_(stream, state) { - var len = state.length; - while (!state.reading && !state.flowing && !state.ended && state.length < state.highWaterMark) { - debug('maybeReadMore read 0'); - stream.read(0); - if (len === state.length) - // didn't get any data, stop spinning. - break;else len = state.length; - } - state.readingMore = false; -} - -// abstract method. to be overridden in specific implementation classes. -// call cb(er, data) where data is <= n in length. -// for virtual (non-string, non-buffer) streams, "length" is somewhat -// arbitrary, and perhaps not very meaningful. -Readable.prototype._read = function (n) { - this.emit('error', new Error('not implemented')); -}; - -Readable.prototype.pipe = function (dest, pipeOpts) { - var src = this; - var state = this._readableState; - - switch (state.pipesCount) { - case 0: - state.pipes = dest; - break; - case 1: - state.pipes = [state.pipes, dest]; - break; - default: - state.pipes.push(dest); - break; - } - state.pipesCount += 1; - debug('pipe count=%d opts=%j', state.pipesCount, pipeOpts); - - var doEnd = (!pipeOpts || pipeOpts.end !== false); - - var endFn = doEnd ? onend : cleanup; - if (state.endEmitted) nextTick(endFn);else src.once('end', endFn); - - dest.on('unpipe', onunpipe); - function onunpipe(readable) { - debug('onunpipe'); - if (readable === src) { - cleanup(); - } - } - - function onend() { - debug('onend'); - dest.end(); - } - - // when the dest drains, it reduces the awaitDrain counter - // on the source. This would be more elegant with a .once() - // handler in flow(), but adding and removing repeatedly is - // too slow. - var ondrain = pipeOnDrain(src); - dest.on('drain', ondrain); - - var cleanedUp = false; - function cleanup() { - debug('cleanup'); - // cleanup event handlers once the pipe is broken - dest.removeListener('close', onclose); - dest.removeListener('finish', onfinish); - dest.removeListener('drain', ondrain); - dest.removeListener('error', onerror); - dest.removeListener('unpipe', onunpipe); - src.removeListener('end', onend); - src.removeListener('end', cleanup); - src.removeListener('data', ondata); - - cleanedUp = true; - - // if the reader is waiting for a drain event from this - // specific writer, then it would cause it to never start - // flowing again. - // So, if this is awaiting a drain, then we just call it now. - // If we don't know, then assume that we are waiting for one. - if (state.awaitDrain && (!dest._writableState || dest._writableState.needDrain)) ondrain(); - } - - // If the user pushes more data while we're writing to dest then we'll end up - // in ondata again. However, we only want to increase awaitDrain once because - // dest will only emit one 'drain' event for the multiple writes. - // => Introduce a guard on increasing awaitDrain. - var increasedAwaitDrain = false; - src.on('data', ondata); - function ondata(chunk) { - debug('ondata'); - increasedAwaitDrain = false; - var ret = dest.write(chunk); - if (false === ret && !increasedAwaitDrain) { - // If the user unpiped during `dest.write()`, it is possible - // to get stuck in a permanently paused state if that write - // also returned false. - // => Check whether `dest` is still a piping destination. - if ((state.pipesCount === 1 && state.pipes === dest || state.pipesCount > 1 && indexOf(state.pipes, dest) !== -1) && !cleanedUp) { - debug('false write response, pause', src._readableState.awaitDrain); - src._readableState.awaitDrain++; - increasedAwaitDrain = true; - } - src.pause(); - } - } - - // if the dest has an error, then stop piping into it. - // however, don't suppress the throwing behavior for this. - function onerror(er) { - debug('onerror', er); - unpipe(); - dest.removeListener('error', onerror); - if (listenerCount(dest, 'error') === 0) dest.emit('error', er); - } - - // Make sure our error handler is attached before userland ones. - prependListener(dest, 'error', onerror); - - // Both close and finish should trigger unpipe, but only once. - function onclose() { - dest.removeListener('finish', onfinish); - unpipe(); - } - dest.once('close', onclose); - function onfinish() { - debug('onfinish'); - dest.removeListener('close', onclose); - unpipe(); - } - dest.once('finish', onfinish); - - function unpipe() { - debug('unpipe'); - src.unpipe(dest); - } - - // tell the dest that it's being piped to - dest.emit('pipe', src); - - // start the flow if it hasn't been started already. - if (!state.flowing) { - debug('pipe resume'); - src.resume(); - } - - return dest; -}; - -function pipeOnDrain(src) { - return function () { - var state = src._readableState; - debug('pipeOnDrain', state.awaitDrain); - if (state.awaitDrain) state.awaitDrain--; - if (state.awaitDrain === 0 && src.listeners('data').length) { - state.flowing = true; - flow(src); - } - }; -} - -Readable.prototype.unpipe = function (dest) { - var state = this._readableState; - - // if we're not piping anywhere, then do nothing. - if (state.pipesCount === 0) return this; - - // just one destination. most common case. - if (state.pipesCount === 1) { - // passed in one, but it's not the right one. - if (dest && dest !== state.pipes) return this; - - if (!dest) dest = state.pipes; - - // got a match. - state.pipes = null; - state.pipesCount = 0; - state.flowing = false; - if (dest) dest.emit('unpipe', this); - return this; - } - - // slow case. multiple pipe destinations. - - if (!dest) { - // remove all. - var dests = state.pipes; - var len = state.pipesCount; - state.pipes = null; - state.pipesCount = 0; - state.flowing = false; - - for (var _i = 0; _i < len; _i++) { - dests[_i].emit('unpipe', this); - }return this; - } - - // try to find the right one. - var i = indexOf(state.pipes, dest); - if (i === -1) return this; - - state.pipes.splice(i, 1); - state.pipesCount -= 1; - if (state.pipesCount === 1) state.pipes = state.pipes[0]; - - dest.emit('unpipe', this); - - return this; -}; - -// set up data events if they are asked for -// Ensure readable listeners eventually get something -Readable.prototype.on = function (ev, fn) { - var res = EventEmitter.prototype.on.call(this, ev, fn); - - if (ev === 'data') { - // Start flowing on next tick if stream isn't explicitly paused - if (this._readableState.flowing !== false) this.resume(); - } else if (ev === 'readable') { - var state = this._readableState; - if (!state.endEmitted && !state.readableListening) { - state.readableListening = state.needReadable = true; - state.emittedReadable = false; - if (!state.reading) { - nextTick(nReadingNextTick, this); - } else if (state.length) { - emitReadable(this); - } - } - } - - return res; -}; -Readable.prototype.addListener = Readable.prototype.on; - -function nReadingNextTick(self) { - debug('readable nexttick read 0'); - self.read(0); -} - -// pause() and resume() are remnants of the legacy readable stream API -// If the user uses them, then switch into old mode. -Readable.prototype.resume = function () { - var state = this._readableState; - if (!state.flowing) { - debug('resume'); - state.flowing = true; - resume(this, state); - } - return this; -}; - -function resume(stream, state) { - if (!state.resumeScheduled) { - state.resumeScheduled = true; - nextTick(resume_, stream, state); - } -} - -function resume_(stream, state) { - if (!state.reading) { - debug('resume read 0'); - stream.read(0); - } - - state.resumeScheduled = false; - state.awaitDrain = 0; - stream.emit('resume'); - flow(stream); - if (state.flowing && !state.reading) stream.read(0); -} - -Readable.prototype.pause = function () { - debug('call pause flowing=%j', this._readableState.flowing); - if (false !== this._readableState.flowing) { - debug('pause'); - this._readableState.flowing = false; - this.emit('pause'); - } - return this; -}; - -function flow(stream) { - var state = stream._readableState; - debug('flow', state.flowing); - while (state.flowing && stream.read() !== null) {} -} - -// wrap an old-style stream as the async data source. -// This is *not* part of the readable stream interface. -// It is an ugly unfortunate mess of history. -Readable.prototype.wrap = function (stream) { - var state = this._readableState; - var paused = false; - - var self = this; - stream.on('end', function () { - debug('wrapped end'); - if (state.decoder && !state.ended) { - var chunk = state.decoder.end(); - if (chunk && chunk.length) self.push(chunk); - } - - self.push(null); - }); - - stream.on('data', function (chunk) { - debug('wrapped data'); - if (state.decoder) chunk = state.decoder.write(chunk); - - // don't skip over falsy values in objectMode - if (state.objectMode && (chunk === null || chunk === undefined)) return;else if (!state.objectMode && (!chunk || !chunk.length)) return; - - var ret = self.push(chunk); - if (!ret) { - paused = true; - stream.pause(); - } - }); - - // proxy all the other methods. - // important when wrapping filters and duplexes. - for (var i in stream) { - if (this[i] === undefined && typeof stream[i] === 'function') { - this[i] = function (method) { - return function () { - return stream[method].apply(stream, arguments); - }; - }(i); - } - } - - // proxy certain important events. - var events = ['error', 'close', 'destroy', 'pause', 'resume']; - forEach(events, function (ev) { - stream.on(ev, self.emit.bind(self, ev)); - }); - - // when we try to consume some more bytes, simply unpause the - // underlying stream. - self._read = function (n) { - debug('wrapped _read', n); - if (paused) { - paused = false; - stream.resume(); - } - }; - - return self; -}; - -// exposed for testing purposes only. -Readable._fromList = fromList; - -// Pluck off n bytes from an array of buffers. -// Length is the combined lengths of all the buffers in the list. -// This function is designed to be inlinable, so please take care when making -// changes to the function body. -function fromList(n, state) { - // nothing buffered - if (state.length === 0) return null; - - var ret; - if (state.objectMode) ret = state.buffer.shift();else if (!n || n >= state.length) { - // read it all, truncate the list - if (state.decoder) ret = state.buffer.join('');else if (state.buffer.length === 1) ret = state.buffer.head.data;else ret = state.buffer.concat(state.length); - state.buffer.clear(); - } else { - // read part of list - ret = fromListPartial(n, state.buffer, state.decoder); - } - - return ret; -} - -// Extracts only enough buffered data to satisfy the amount requested. -// This function is designed to be inlinable, so please take care when making -// changes to the function body. -function fromListPartial(n, list, hasStrings) { - var ret; - if (n < list.head.data.length) { - // slice is the same for buffers and strings - ret = list.head.data.slice(0, n); - list.head.data = list.head.data.slice(n); - } else if (n === list.head.data.length) { - // first chunk is a perfect match - ret = list.shift(); - } else { - // result spans more than one buffer - ret = hasStrings ? copyFromBufferString(n, list) : copyFromBuffer(n, list); - } - return ret; -} - -// Copies a specified amount of characters from the list of buffered data -// chunks. -// This function is designed to be inlinable, so please take care when making -// changes to the function body. -function copyFromBufferString(n, list) { - var p = list.head; - var c = 1; - var ret = p.data; - n -= ret.length; - while (p = p.next) { - var str = p.data; - var nb = n > str.length ? str.length : n; - if (nb === str.length) ret += str;else ret += str.slice(0, n); - n -= nb; - if (n === 0) { - if (nb === str.length) { - ++c; - if (p.next) list.head = p.next;else list.head = list.tail = null; - } else { - list.head = p; - p.data = str.slice(nb); - } - break; - } - ++c; - } - list.length -= c; - return ret; -} - -// Copies a specified amount of bytes from the list of buffered data chunks. -// This function is designed to be inlinable, so please take care when making -// changes to the function body. -function copyFromBuffer(n, list) { - var ret = Buffer.allocUnsafe(n); - var p = list.head; - var c = 1; - p.data.copy(ret); - n -= p.data.length; - while (p = p.next) { - var buf = p.data; - var nb = n > buf.length ? buf.length : n; - buf.copy(ret, ret.length - n, 0, nb); - n -= nb; - if (n === 0) { - if (nb === buf.length) { - ++c; - if (p.next) list.head = p.next;else list.head = list.tail = null; - } else { - list.head = p; - p.data = buf.slice(nb); - } - break; - } - ++c; - } - list.length -= c; - return ret; -} - -function endReadable(stream) { - var state = stream._readableState; - - // If we get here before consuming all the bytes, then that is a - // bug in node. Should never happen. - if (state.length > 0) throw new Error('"endReadable()" called on non-empty stream'); - - if (!state.endEmitted) { - state.ended = true; - nextTick(endReadableNT, state, stream); - } -} - -function endReadableNT(state, stream) { - // Check that we didn't get one last unshift. - if (!state.endEmitted && state.length === 0) { - state.endEmitted = true; - stream.readable = false; - stream.emit('end'); - } -} - -function forEach(xs, f) { - for (var i = 0, l = xs.length; i < l; i++) { - f(xs[i], i); - } -} - -function indexOf(xs, x) { - for (var i = 0, l = xs.length; i < l; i++) { - if (xs[i] === x) return i; - } - return -1; -} - -// A bit simpler than readable streams. -Writable.WritableState = WritableState; -inherits$1(Writable, EventEmitter); - -function nop() {} - -function WriteReq(chunk, encoding, cb) { - this.chunk = chunk; - this.encoding = encoding; - this.callback = cb; - this.next = null; -} - -function WritableState(options, stream) { - Object.defineProperty(this, 'buffer', { - get: deprecate(function () { - return this.getBuffer(); - }, '_writableState.buffer is deprecated. Use _writableState.getBuffer ' + 'instead.') - }); - options = options || {}; - - // object stream flag to indicate whether or not this stream - // contains buffers or objects. - this.objectMode = !!options.objectMode; - - if (stream instanceof Duplex) this.objectMode = this.objectMode || !!options.writableObjectMode; - - // the point at which write() starts returning false - // Note: 0 is a valid value, means that we always return false if - // the entire buffer is not flushed immediately on write() - var hwm = options.highWaterMark; - var defaultHwm = this.objectMode ? 16 : 16 * 1024; - this.highWaterMark = hwm || hwm === 0 ? hwm : defaultHwm; - - // cast to ints. - this.highWaterMark = ~ ~this.highWaterMark; - - this.needDrain = false; - // at the start of calling end() - this.ending = false; - // when end() has been called, and returned - this.ended = false; - // when 'finish' is emitted - this.finished = false; - - // should we decode strings into buffers before passing to _write? - // this is here so that some node-core streams can optimize string - // handling at a lower level. - var noDecode = options.decodeStrings === false; - this.decodeStrings = !noDecode; - - // Crypto is kind of old and crusty. Historically, its default string - // encoding is 'binary' so we have to make this configurable. - // Everything else in the universe uses 'utf8', though. - this.defaultEncoding = options.defaultEncoding || 'utf8'; - - // not an actual buffer we keep track of, but a measurement - // of how much we're waiting to get pushed to some underlying - // socket or file. - this.length = 0; - - // a flag to see when we're in the middle of a write. - this.writing = false; - - // when true all writes will be buffered until .uncork() call - this.corked = 0; - - // a flag to be able to tell if the onwrite cb is called immediately, - // or on a later tick. We set this to true at first, because any - // actions that shouldn't happen until "later" should generally also - // not happen before the first write call. - this.sync = true; - - // a flag to know if we're processing previously buffered items, which - // may call the _write() callback in the same tick, so that we don't - // end up in an overlapped onwrite situation. - this.bufferProcessing = false; - - // the callback that's passed to _write(chunk,cb) - this.onwrite = function (er) { - onwrite(stream, er); - }; - - // the callback that the user supplies to write(chunk,encoding,cb) - this.writecb = null; - - // the amount that is being written when _write is called. - this.writelen = 0; - - this.bufferedRequest = null; - this.lastBufferedRequest = null; - - // number of pending user-supplied write callbacks - // this must be 0 before 'finish' can be emitted - this.pendingcb = 0; - - // emit prefinish if the only thing we're waiting for is _write cbs - // This is relevant for synchronous Transform streams - this.prefinished = false; - - // True if the error was already emitted and should not be thrown again - this.errorEmitted = false; - - // count buffered requests - this.bufferedRequestCount = 0; - - // allocate the first CorkedRequest, there is always - // one allocated and free to use, and we maintain at most two - this.corkedRequestsFree = new CorkedRequest(this); -} - -WritableState.prototype.getBuffer = function writableStateGetBuffer() { - var current = this.bufferedRequest; - var out = []; - while (current) { - out.push(current); - current = current.next; - } - return out; -}; -function Writable(options) { - - // Writable ctor is applied to Duplexes, though they're not - // instanceof Writable, they're instanceof Readable. - if (!(this instanceof Writable) && !(this instanceof Duplex)) return new Writable(options); - - this._writableState = new WritableState(options, this); - - // legacy. - this.writable = true; - - if (options) { - if (typeof options.write === 'function') this._write = options.write; - - if (typeof options.writev === 'function') this._writev = options.writev; - } - - EventEmitter.call(this); -} - -// Otherwise people can pipe Writable streams, which is just wrong. -Writable.prototype.pipe = function () { - this.emit('error', new Error('Cannot pipe, not readable')); -}; - -function writeAfterEnd(stream, cb) { - var er = new Error('write after end'); - // TODO: defer error events consistently everywhere, not just the cb - stream.emit('error', er); - nextTick(cb, er); -} - -// If we get something that is not a buffer, string, null, or undefined, -// and we're not in objectMode, then that's an error. -// Otherwise stream chunks are all considered to be of length=1, and the -// watermarks determine how many objects to keep in the buffer, rather than -// how many bytes or characters. -function validChunk(stream, state, chunk, cb) { - var valid = true; - var er = false; - // Always throw error if a null is written - // if we are not in object mode then throw - // if it is not a buffer, string, or undefined. - if (chunk === null) { - er = new TypeError('May not write null values to stream'); - } else if (!Buffer.isBuffer(chunk) && typeof chunk !== 'string' && chunk !== undefined && !state.objectMode) { - er = new TypeError('Invalid non-string/buffer chunk'); - } - if (er) { - stream.emit('error', er); - nextTick(cb, er); - valid = false; - } - return valid; -} - -Writable.prototype.write = function (chunk, encoding, cb) { - var state = this._writableState; - var ret = false; - - if (typeof encoding === 'function') { - cb = encoding; - encoding = null; - } - - if (Buffer.isBuffer(chunk)) encoding = 'buffer';else if (!encoding) encoding = state.defaultEncoding; - - if (typeof cb !== 'function') cb = nop; - - if (state.ended) writeAfterEnd(this, cb);else if (validChunk(this, state, chunk, cb)) { - state.pendingcb++; - ret = writeOrBuffer(this, state, chunk, encoding, cb); - } - - return ret; -}; - -Writable.prototype.cork = function () { - var state = this._writableState; - - state.corked++; -}; - -Writable.prototype.uncork = function () { - var state = this._writableState; - - if (state.corked) { - state.corked--; - - if (!state.writing && !state.corked && !state.finished && !state.bufferProcessing && state.bufferedRequest) clearBuffer(this, state); - } -}; - -Writable.prototype.setDefaultEncoding = function setDefaultEncoding(encoding) { - // node::ParseEncoding() requires lower case. - if (typeof encoding === 'string') encoding = encoding.toLowerCase(); - if (!(['hex', 'utf8', 'utf-8', 'ascii', 'binary', 'base64', 'ucs2', 'ucs-2', 'utf16le', 'utf-16le', 'raw'].indexOf((encoding + '').toLowerCase()) > -1)) throw new TypeError('Unknown encoding: ' + encoding); - this._writableState.defaultEncoding = encoding; - return this; -}; - -function decodeChunk(state, chunk, encoding) { - if (!state.objectMode && state.decodeStrings !== false && typeof chunk === 'string') { - chunk = Buffer.from(chunk, encoding); - } - return chunk; -} - -// if we're already writing something, then just put this -// in the queue, and wait our turn. Otherwise, call _write -// If we return false, then we need a drain event, so set that flag. -function writeOrBuffer(stream, state, chunk, encoding, cb) { - chunk = decodeChunk(state, chunk, encoding); - - if (Buffer.isBuffer(chunk)) encoding = 'buffer'; - var len = state.objectMode ? 1 : chunk.length; - - state.length += len; - - var ret = state.length < state.highWaterMark; - // we must ensure that previous needDrain will not be reset to false. - if (!ret) state.needDrain = true; - - if (state.writing || state.corked) { - var last = state.lastBufferedRequest; - state.lastBufferedRequest = new WriteReq(chunk, encoding, cb); - if (last) { - last.next = state.lastBufferedRequest; - } else { - state.bufferedRequest = state.lastBufferedRequest; - } - state.bufferedRequestCount += 1; - } else { - doWrite(stream, state, false, len, chunk, encoding, cb); - } - - return ret; -} - -function doWrite(stream, state, writev, len, chunk, encoding, cb) { - state.writelen = len; - state.writecb = cb; - state.writing = true; - state.sync = true; - if (writev) stream._writev(chunk, state.onwrite);else stream._write(chunk, encoding, state.onwrite); - state.sync = false; -} - -function onwriteError(stream, state, sync, er, cb) { - --state.pendingcb; - if (sync) nextTick(cb, er);else cb(er); - - stream._writableState.errorEmitted = true; - stream.emit('error', er); -} - -function onwriteStateUpdate(state) { - state.writing = false; - state.writecb = null; - state.length -= state.writelen; - state.writelen = 0; -} - -function onwrite(stream, er) { - var state = stream._writableState; - var sync = state.sync; - var cb = state.writecb; - - onwriteStateUpdate(state); - - if (er) onwriteError(stream, state, sync, er, cb);else { - // Check if we're actually ready to finish, but don't emit yet - var finished = needFinish(state); - - if (!finished && !state.corked && !state.bufferProcessing && state.bufferedRequest) { - clearBuffer(stream, state); - } - - if (sync) { - /**/ - nextTick(afterWrite, stream, state, finished, cb); - /**/ - } else { - afterWrite(stream, state, finished, cb); - } - } -} - -function afterWrite(stream, state, finished, cb) { - if (!finished) onwriteDrain(stream, state); - state.pendingcb--; - cb(); - finishMaybe(stream, state); -} - -// Must force callback to be called on nextTick, so that we don't -// emit 'drain' before the write() consumer gets the 'false' return -// value, and has a chance to attach a 'drain' listener. -function onwriteDrain(stream, state) { - if (state.length === 0 && state.needDrain) { - state.needDrain = false; - stream.emit('drain'); - } -} - -// if there's something in the buffer waiting, then process it -function clearBuffer(stream, state) { - state.bufferProcessing = true; - var entry = state.bufferedRequest; - - if (stream._writev && entry && entry.next) { - // Fast case, write everything using _writev() - var l = state.bufferedRequestCount; - var buffer = new Array(l); - var holder = state.corkedRequestsFree; - holder.entry = entry; - - var count = 0; - while (entry) { - buffer[count] = entry; - entry = entry.next; - count += 1; - } - - doWrite(stream, state, true, state.length, buffer, '', holder.finish); - - // doWrite is almost always async, defer these to save a bit of time - // as the hot path ends with doWrite - state.pendingcb++; - state.lastBufferedRequest = null; - if (holder.next) { - state.corkedRequestsFree = holder.next; - holder.next = null; - } else { - state.corkedRequestsFree = new CorkedRequest(state); - } - } else { - // Slow case, write chunks one-by-one - while (entry) { - var chunk = entry.chunk; - var encoding = entry.encoding; - var cb = entry.callback; - var len = state.objectMode ? 1 : chunk.length; - - doWrite(stream, state, false, len, chunk, encoding, cb); - entry = entry.next; - // if we didn't call the onwrite immediately, then - // it means that we need to wait until it does. - // also, that means that the chunk and cb are currently - // being processed, so move the buffer counter past them. - if (state.writing) { - break; - } - } - - if (entry === null) state.lastBufferedRequest = null; - } - - state.bufferedRequestCount = 0; - state.bufferedRequest = entry; - state.bufferProcessing = false; -} - -Writable.prototype._write = function (chunk, encoding, cb) { - cb(new Error('not implemented')); -}; - -Writable.prototype._writev = null; - -Writable.prototype.end = function (chunk, encoding, cb) { - var state = this._writableState; - - if (typeof chunk === 'function') { - cb = chunk; - chunk = null; - encoding = null; - } else if (typeof encoding === 'function') { - cb = encoding; - encoding = null; - } - - if (chunk !== null && chunk !== undefined) this.write(chunk, encoding); - - // .end() fully uncorks - if (state.corked) { - state.corked = 1; - this.uncork(); - } - - // ignore unnecessary end() calls. - if (!state.ending && !state.finished) endWritable(this, state, cb); -}; - -function needFinish(state) { - return state.ending && state.length === 0 && state.bufferedRequest === null && !state.finished && !state.writing; -} - -function prefinish(stream, state) { - if (!state.prefinished) { - state.prefinished = true; - stream.emit('prefinish'); - } -} - -function finishMaybe(stream, state) { - var need = needFinish(state); - if (need) { - if (state.pendingcb === 0) { - prefinish(stream, state); - state.finished = true; - stream.emit('finish'); - } else { - prefinish(stream, state); - } - } - return need; -} - -function endWritable(stream, state, cb) { - state.ending = true; - finishMaybe(stream, state); - if (cb) { - if (state.finished) nextTick(cb);else stream.once('finish', cb); - } - state.ended = true; - stream.writable = false; -} - -// It seems a linked list but it is not -// there will be only 2 of these for each stream -function CorkedRequest(state) { - var _this = this; - - this.next = null; - this.entry = null; - - this.finish = function (err) { - var entry = _this.entry; - _this.entry = null; - while (entry) { - var cb = entry.callback; - state.pendingcb--; - cb(err); - entry = entry.next; - } - if (state.corkedRequestsFree) { - state.corkedRequestsFree.next = _this; - } else { - state.corkedRequestsFree = _this; - } - }; -} - -inherits$1(Duplex, Readable); - -var keys = Object.keys(Writable.prototype); -for (var v = 0; v < keys.length; v++) { - var method = keys[v]; - if (!Duplex.prototype[method]) Duplex.prototype[method] = Writable.prototype[method]; -} -function Duplex(options) { - if (!(this instanceof Duplex)) return new Duplex(options); - - Readable.call(this, options); - Writable.call(this, options); - - if (options && options.readable === false) this.readable = false; - - if (options && options.writable === false) this.writable = false; - - this.allowHalfOpen = true; - if (options && options.allowHalfOpen === false) this.allowHalfOpen = false; - - this.once('end', onend); -} - -// the no-half-open enforcer -function onend() { - // if we allow half-open state, or if the writable side ended, - // then we're ok. - if (this.allowHalfOpen || this._writableState.ended) return; - - // no more data can be written. - // But allow more writes to happen in this tick. - nextTick(onEndNT, this); -} - -function onEndNT(self) { - self.end(); -} - -// a transform stream is a readable/writable stream where you do -inherits$1(Transform, Duplex); - -function TransformState(stream) { - this.afterTransform = function (er, data) { - return afterTransform(stream, er, data); - }; - - this.needTransform = false; - this.transforming = false; - this.writecb = null; - this.writechunk = null; - this.writeencoding = null; -} - -function afterTransform(stream, er, data) { - var ts = stream._transformState; - ts.transforming = false; - - var cb = ts.writecb; - - if (!cb) return stream.emit('error', new Error('no writecb in Transform class')); - - ts.writechunk = null; - ts.writecb = null; - - if (data !== null && data !== undefined) stream.push(data); - - cb(er); - - var rs = stream._readableState; - rs.reading = false; - if (rs.needReadable || rs.length < rs.highWaterMark) { - stream._read(rs.highWaterMark); - } -} -function Transform(options) { - if (!(this instanceof Transform)) return new Transform(options); - - Duplex.call(this, options); - - this._transformState = new TransformState(this); - - // when the writable side finishes, then flush out anything remaining. - var stream = this; - - // start out asking for a readable event once data is transformed. - this._readableState.needReadable = true; - - // we have implemented the _read method, and done the other things - // that Readable wants before the first _read call, so unset the - // sync guard flag. - this._readableState.sync = false; - - if (options) { - if (typeof options.transform === 'function') this._transform = options.transform; - - if (typeof options.flush === 'function') this._flush = options.flush; - } - - this.once('prefinish', function () { - if (typeof this._flush === 'function') this._flush(function (er) { - done(stream, er); - });else done(stream); - }); -} - -Transform.prototype.push = function (chunk, encoding) { - this._transformState.needTransform = false; - return Duplex.prototype.push.call(this, chunk, encoding); -}; - -// This is the part where you do stuff! -// override this function in implementation classes. -// 'chunk' is an input chunk. -// -// Call `push(newChunk)` to pass along transformed output -// to the readable side. You may call 'push' zero or more times. -// -// Call `cb(err)` when you are done with this chunk. If you pass -// an error, then that'll put the hurt on the whole operation. If you -// never call cb(), then you'll never get another chunk. -Transform.prototype._transform = function (chunk, encoding, cb) { - throw new Error('Not implemented'); -}; - -Transform.prototype._write = function (chunk, encoding, cb) { - var ts = this._transformState; - ts.writecb = cb; - ts.writechunk = chunk; - ts.writeencoding = encoding; - if (!ts.transforming) { - var rs = this._readableState; - if (ts.needTransform || rs.needReadable || rs.length < rs.highWaterMark) this._read(rs.highWaterMark); - } -}; - -// Doesn't matter what the args are here. -// _transform does all the work. -// That we got here means that the readable side wants more data. -Transform.prototype._read = function (n) { - var ts = this._transformState; - - if (ts.writechunk !== null && ts.writecb && !ts.transforming) { - ts.transforming = true; - this._transform(ts.writechunk, ts.writeencoding, ts.afterTransform); - } else { - // mark that we need a transform, so that any data that comes in - // will get processed, now that we've asked for it. - ts.needTransform = true; - } -}; - -function done(stream, er) { - if (er) return stream.emit('error', er); - - // if there's nothing in the write buffer, then that means - // that nothing more will ever be provided - var ws = stream._writableState; - var ts = stream._transformState; - - if (ws.length) throw new Error('Calling transform done when ws.length != 0'); - - if (ts.transforming) throw new Error('Calling transform done when still transforming'); - - return stream.push(null); -} - -inherits$1(PassThrough, Transform); -function PassThrough(options) { - if (!(this instanceof PassThrough)) return new PassThrough(options); - - Transform.call(this, options); -} - -PassThrough.prototype._transform = function (chunk, encoding, cb) { - cb(null, chunk); -}; - -inherits$1(Stream, EventEmitter); -Stream.Readable = Readable; -Stream.Writable = Writable; -Stream.Duplex = Duplex; -Stream.Transform = Transform; -Stream.PassThrough = PassThrough; - -// Backwards-compat with node 0.4.x -Stream.Stream = Stream; - -// old-style streams. Note that the pipe method (the only relevant -// part of this class) is overridden in the Readable class. - -function Stream() { - EventEmitter.call(this); -} - -Stream.prototype.pipe = function(dest, options) { - var source = this; - - function ondata(chunk) { - if (dest.writable) { - if (false === dest.write(chunk) && source.pause) { - source.pause(); - } - } - } - - source.on('data', ondata); - - function ondrain() { - if (source.readable && source.resume) { - source.resume(); - } - } - - dest.on('drain', ondrain); - - // If the 'end' option is not supplied, dest.end() will be called when - // source gets the 'end' or 'close' events. Only dest.end() once. - if (!dest._isStdio && (!options || options.end !== false)) { - source.on('end', onend); - source.on('close', onclose); - } - - var didOnEnd = false; - function onend() { - if (didOnEnd) return; - didOnEnd = true; - - dest.end(); - } - - - function onclose() { - if (didOnEnd) return; - didOnEnd = true; - - if (typeof dest.destroy === 'function') dest.destroy(); - } - - // don't leave dangling pipes when there are errors. - function onerror(er) { - cleanup(); - if (EventEmitter.listenerCount(this, 'error') === 0) { - throw er; // Unhandled stream error in pipe. - } - } - - source.on('error', onerror); - dest.on('error', onerror); - - // remove all the event listeners that were added. - function cleanup() { - source.removeListener('data', ondata); - dest.removeListener('drain', ondrain); - - source.removeListener('end', onend); - source.removeListener('close', onclose); - - source.removeListener('error', onerror); - dest.removeListener('error', onerror); - - source.removeListener('end', cleanup); - source.removeListener('close', cleanup); - - dest.removeListener('close', cleanup); - } - - source.on('end', cleanup); - source.on('close', cleanup); - - dest.on('close', cleanup); - - dest.emit('pipe', source); - - // Allow for unix-like usage: A.pipe(B).pipe(C) - return dest; -}; - -const bom_utf8 = Buffer.from([239, 187, 191]); - -class CsvError extends Error { - constructor(code, message, ...contexts) { - if(Array.isArray(message)) message = message.join(' '); - super(message); - if(Error.captureStackTrace !== undefined){ - Error.captureStackTrace(this, CsvError); - } - this.code = code; - for(const context of contexts){ - for(const key in context){ - const value = context[key]; - this[key] = isBuffer(value) ? value.toString() : value == null ? value : JSON.parse(JSON.stringify(value)); - } - } - } -} - -const isObject = function(obj){ - return typeof obj === 'object' && obj !== null && ! Array.isArray(obj); -}; - -const underscore = function(str){ - return str.replace(/([A-Z])/g, function(_, match){ - return '_' + match.toLowerCase(); - }); -}; - // Lodash implementation of `get` const charCodeOfDot = '.'.charCodeAt(0); @@ -5072,455 +2040,473 @@ const get = function(object, path){ return (index && index === length) ? object : undefined; }; -class Stringifier extends Transform { - constructor(opts = {}){ - super({...{writableObjectMode: true}, ...opts}); - const options = {}; - let err; - // Merge with user options - for(const opt in opts){ - options[underscore(opt)] = opts[opt]; - } - if((err = this.normalize(options)) !== undefined) throw err; - switch(options.record_delimiter){ - case 'auto': - options.record_delimiter = null; - break; - case 'unix': - options.record_delimiter = "\n"; - break; - case 'mac': - options.record_delimiter = "\r"; - break; - case 'windows': - options.record_delimiter = "\r\n"; - break; - case 'ascii': - options.record_delimiter = "\u001e"; - break; - case 'unicode': - options.record_delimiter = "\u2028"; - break; - } - // Expose options - this.options = options; - // Internal state - this.state = { - stop: false - }; - // Information - this.info = { - records: 0 - }; - } - normalize(options){ - // Normalize option `bom` - if(options.bom === undefined || options.bom === null || options.bom === false){ - options.bom = false; - }else if(options.bom !== true){ - return new CsvError('CSV_OPTION_BOOLEAN_INVALID_TYPE', [ - 'option `bom` is optional and must be a boolean value,', - `got ${JSON.stringify(options.bom)}` - ]); - } - // Normalize option `delimiter` - if(options.delimiter === undefined || options.delimiter === null){ - options.delimiter = ','; - }else if(isBuffer(options.delimiter)){ - options.delimiter = options.delimiter.toString(); - }else if(typeof options.delimiter !== 'string'){ - return new CsvError('CSV_OPTION_DELIMITER_INVALID_TYPE', [ - 'option `delimiter` must be a buffer or a string,', - `got ${JSON.stringify(options.delimiter)}` - ]); - } - // Normalize option `quote` - if(options.quote === undefined || options.quote === null){ - options.quote = '"'; - }else if(options.quote === true){ - options.quote = '"'; - }else if(options.quote === false){ - options.quote = ''; - }else if (isBuffer(options.quote)){ - options.quote = options.quote.toString(); - }else if(typeof options.quote !== 'string'){ - return new CsvError('CSV_OPTION_QUOTE_INVALID_TYPE', [ - 'option `quote` must be a boolean, a buffer or a string,', - `got ${JSON.stringify(options.quote)}` - ]); - } - // Normalize option `quoted` - if(options.quoted === undefined || options.quoted === null){ - options.quoted = false; - } - // Normalize option `quoted_empty` - if(options.quoted_empty === undefined || options.quoted_empty === null){ - options.quoted_empty = undefined; - } - // Normalize option `quoted_match` - if(options.quoted_match === undefined || options.quoted_match === null || options.quoted_match === false){ - options.quoted_match = null; - }else if(!Array.isArray(options.quoted_match)){ - options.quoted_match = [options.quoted_match]; +const is_object = function(obj){ + return typeof obj === 'object' && obj !== null && ! Array.isArray(obj); +}; + +const normalize_columns = function(columns){ + if(columns === undefined || columns === null){ + return [undefined, undefined]; + } + if(typeof columns !== 'object'){ + return [Error('Invalid option "columns": expect an array or an object')]; + } + if(!Array.isArray(columns)){ + const newcolumns = []; + for(const k in columns){ + newcolumns.push({ + key: k, + header: columns[k] + }); } - if(options.quoted_match){ - for(const quoted_match of options.quoted_match){ - const isString = typeof quoted_match === 'string'; - const isRegExp = quoted_match instanceof RegExp; - if(!isString && !isRegExp){ - return Error(`Invalid Option: quoted_match must be a string or a regex, got ${JSON.stringify(quoted_match)}`); + columns = newcolumns; + }else { + const newcolumns = []; + for(const column of columns){ + if(typeof column === 'string'){ + newcolumns.push({ + key: column, + header: column + }); + }else if(typeof column === 'object' && column !== null && !Array.isArray(column)){ + if(!column.key){ + return [Error('Invalid column definition: property "key" is required')]; + } + if(column.header === undefined){ + column.header = column.key; } + newcolumns.push(column); + }else { + return [Error('Invalid column definition: expect a string or an object')]; } } - // Normalize option `quoted_string` - if(options.quoted_string === undefined || options.quoted_string === null){ - options.quoted_string = false; - } - // Normalize option `eof` - if(options.eof === undefined || options.eof === null){ - options.eof = true; - } - // Normalize option `escape` - if(options.escape === undefined || options.escape === null){ - options.escape = '"'; - }else if(isBuffer(options.escape)){ - options.escape = options.escape.toString(); - }else if(typeof options.escape !== 'string'){ - return Error(`Invalid Option: escape must be a buffer or a string, got ${JSON.stringify(options.escape)}`); - } - if (options.escape.length > 1){ - return Error(`Invalid Option: escape must be one character, got ${options.escape.length} characters`); - } - // Normalize option `header` - if(options.header === undefined || options.header === null){ - options.header = false; - } - // Normalize option `columns` - const [err, columns] = this.normalize_columns(options.columns); - if(err) return err; - options.columns = columns; - // Normalize option `quoted` - if(options.quoted === undefined || options.quoted === null){ - options.quoted = false; - } - // Normalize option `cast` - if(options.cast === undefined || options.cast === null){ - options.cast = {}; - } - // Normalize option cast.bigint - if(options.cast.bigint === undefined || options.cast.bigint === null){ - // Cast boolean to string by default - options.cast.bigint = value => '' + value; - } - // Normalize option cast.boolean - if(options.cast.boolean === undefined || options.cast.boolean === null){ - // Cast boolean to string by default - options.cast.boolean = value => value ? '1' : ''; - } - // Normalize option cast.date - if(options.cast.date === undefined || options.cast.date === null){ - // Cast date to timestamp string by default - options.cast.date = value => '' + value.getTime(); - } - // Normalize option cast.number - if(options.cast.number === undefined || options.cast.number === null){ - // Cast number to string using native casting by default - options.cast.number = value => '' + value; - } - // Normalize option cast.object - if(options.cast.object === undefined || options.cast.object === null){ - // Stringify object as JSON by default - options.cast.object = value => JSON.stringify(value); - } - // Normalize option cast.string - if(options.cast.string === undefined || options.cast.string === null){ - // Leave string untouched - options.cast.string = function(value){return value;}; - } - // Normalize option `record_delimiter` - if(options.record_delimiter === undefined || options.record_delimiter === null){ - options.record_delimiter = '\n'; - }else if(isBuffer(options.record_delimiter)){ - options.record_delimiter = options.record_delimiter.toString(); - }else if(typeof options.record_delimiter !== 'string'){ - return Error(`Invalid Option: record_delimiter must be a buffer or a string, got ${JSON.stringify(options.record_delimiter)}`); - } + columns = newcolumns; } - _transform(chunk, encoding, callback){ - if(this.state.stop === true){ - return; + return [undefined, columns]; +}; + +class CsvError extends Error { + constructor(code, message, ...contexts) { + if(Array.isArray(message)) message = message.join(' '); + super(message); + if(Error.captureStackTrace !== undefined){ + Error.captureStackTrace(this, CsvError); } - const err = this.__transform(chunk); - if(err !== undefined){ - this.state.stop = true; + this.code = code; + for(const context of contexts){ + for(const key in context){ + const value = context[key]; + this[key] = isBuffer(value) ? value.toString() : value == null ? value : JSON.parse(JSON.stringify(value)); + } } - callback(err); } - _flush(callback){ - if(this.state.stop === true){ - // Note, Node.js 12 call flush even after an error, we must prevent - // `callback` from being called in flush without any error. - return; - } - if(this.info.records === 0){ - this.bom(); - const err = this.headers(); - if(err) callback(err); +} + +const underscore = function(str){ + return str.replace(/([A-Z])/g, function(_, match){ + return '_' + match.toLowerCase(); + }); +}; + +const normalize_options = function(opts) { + const options = {}; + // Merge with user options + for(const opt in opts){ + options[underscore(opt)] = opts[opt]; + } + // Normalize option `bom` + if(options.bom === undefined || options.bom === null || options.bom === false){ + options.bom = false; + }else if(options.bom !== true){ + return [new CsvError('CSV_OPTION_BOOLEAN_INVALID_TYPE', [ + 'option `bom` is optional and must be a boolean value,', + `got ${JSON.stringify(options.bom)}` + ])]; + } + // Normalize option `delimiter` + if(options.delimiter === undefined || options.delimiter === null){ + options.delimiter = ','; + }else if(isBuffer(options.delimiter)){ + options.delimiter = options.delimiter.toString(); + }else if(typeof options.delimiter !== 'string'){ + return [new CsvError('CSV_OPTION_DELIMITER_INVALID_TYPE', [ + 'option `delimiter` must be a buffer or a string,', + `got ${JSON.stringify(options.delimiter)}` + ])]; + } + // Normalize option `quote` + if(options.quote === undefined || options.quote === null){ + options.quote = '"'; + }else if(options.quote === true){ + options.quote = '"'; + }else if(options.quote === false){ + options.quote = ''; + }else if (isBuffer(options.quote)){ + options.quote = options.quote.toString(); + }else if(typeof options.quote !== 'string'){ + return [new CsvError('CSV_OPTION_QUOTE_INVALID_TYPE', [ + 'option `quote` must be a boolean, a buffer or a string,', + `got ${JSON.stringify(options.quote)}` + ])]; + } + // Normalize option `quoted` + if(options.quoted === undefined || options.quoted === null){ + options.quoted = false; + } + // Normalize option `quoted_empty` + if(options.quoted_empty === undefined || options.quoted_empty === null){ + options.quoted_empty = undefined; + } + // Normalize option `quoted_match` + if(options.quoted_match === undefined || options.quoted_match === null || options.quoted_match === false){ + options.quoted_match = null; + }else if(!Array.isArray(options.quoted_match)){ + options.quoted_match = [options.quoted_match]; + } + if(options.quoted_match){ + for(const quoted_match of options.quoted_match){ + const isString = typeof quoted_match === 'string'; + const isRegExp = quoted_match instanceof RegExp; + if(!isString && !isRegExp){ + return [Error(`Invalid Option: quoted_match must be a string or a regex, got ${JSON.stringify(quoted_match)}`)]; + } } - callback(); } - __transform(chunk){ - // Chunk validation - if(!Array.isArray(chunk) && typeof chunk !== 'object'){ - return Error(`Invalid Record: expect an array or an object, got ${JSON.stringify(chunk)}`); - } - // Detect columns from the first record - if(this.info.records === 0){ - if(Array.isArray(chunk)){ - if(this.options.header === true && this.options.columns === undefined){ - return Error('Undiscoverable Columns: header option requires column option or object records'); + // Normalize option `quoted_string` + if(options.quoted_string === undefined || options.quoted_string === null){ + options.quoted_string = false; + } + // Normalize option `eof` + if(options.eof === undefined || options.eof === null){ + options.eof = true; + } + // Normalize option `escape` + if(options.escape === undefined || options.escape === null){ + options.escape = '"'; + }else if(isBuffer(options.escape)){ + options.escape = options.escape.toString(); + }else if(typeof options.escape !== 'string'){ + return [Error(`Invalid Option: escape must be a buffer or a string, got ${JSON.stringify(options.escape)}`)]; + } + if (options.escape.length > 1){ + return [Error(`Invalid Option: escape must be one character, got ${options.escape.length} characters`)]; + } + // Normalize option `header` + if(options.header === undefined || options.header === null){ + options.header = false; + } + // Normalize option `columns` + const [errColumns, columns] = normalize_columns(options.columns); + if(errColumns !== undefined) return [errColumns]; + options.columns = columns; + // Normalize option `quoted` + if(options.quoted === undefined || options.quoted === null){ + options.quoted = false; + } + // Normalize option `cast` + if(options.cast === undefined || options.cast === null){ + options.cast = {}; + } + // Normalize option cast.bigint + if(options.cast.bigint === undefined || options.cast.bigint === null){ + // Cast boolean to string by default + options.cast.bigint = value => '' + value; + } + // Normalize option cast.boolean + if(options.cast.boolean === undefined || options.cast.boolean === null){ + // Cast boolean to string by default + options.cast.boolean = value => value ? '1' : ''; + } + // Normalize option cast.date + if(options.cast.date === undefined || options.cast.date === null){ + // Cast date to timestamp string by default + options.cast.date = value => '' + value.getTime(); + } + // Normalize option cast.number + if(options.cast.number === undefined || options.cast.number === null){ + // Cast number to string using native casting by default + options.cast.number = value => '' + value; + } + // Normalize option cast.object + if(options.cast.object === undefined || options.cast.object === null){ + // Stringify object as JSON by default + options.cast.object = value => JSON.stringify(value); + } + // Normalize option cast.string + if(options.cast.string === undefined || options.cast.string === null){ + // Leave string untouched + options.cast.string = function(value){return value;}; + } + // Normalize option `on_record` + if(options.on_record !== undefined && typeof options.on_record !== 'function'){ + return [Error(`Invalid Option: "on_record" must be a function.`)]; + } + // Normalize option `record_delimiter` + if(options.record_delimiter === undefined || options.record_delimiter === null){ + options.record_delimiter = '\n'; + }else if(isBuffer(options.record_delimiter)){ + options.record_delimiter = options.record_delimiter.toString(); + }else if(typeof options.record_delimiter !== 'string'){ + return [Error(`Invalid Option: record_delimiter must be a buffer or a string, got ${JSON.stringify(options.record_delimiter)}`)]; + } + switch(options.record_delimiter){ + case 'auto': + options.record_delimiter = null; + break; + case 'unix': + options.record_delimiter = "\n"; + break; + case 'mac': + options.record_delimiter = "\r"; + break; + case 'windows': + options.record_delimiter = "\r\n"; + break; + case 'ascii': + options.record_delimiter = "\u001e"; + break; + case 'unicode': + options.record_delimiter = "\u2028"; + break; + } + return [undefined, options]; +}; + +const bom_utf8 = Buffer.from([239, 187, 191]); + +const stringifier = function(options, state, info){ + return { + options: options, + state: state, + info: info, + __transform: function(chunk, push){ + // Chunk validation + if(!Array.isArray(chunk) && typeof chunk !== 'object'){ + return Error(`Invalid Record: expect an array or an object, got ${JSON.stringify(chunk)}`); + } + // Detect columns from the first record + if(this.info.records === 0){ + if(Array.isArray(chunk)){ + if(this.options.header === true && this.options.columns === undefined){ + return Error('Undiscoverable Columns: header option requires column option or object records'); + } + }else if(this.options.columns === undefined){ + const [err, columns] = normalize_columns(Object.keys(chunk)); + if(err) return; + this.options.columns = columns; } - }else if(this.options.columns === undefined){ - const [err, columns] = this.normalize_columns(Object.keys(chunk)); - if(err) return; - this.options.columns = columns; } - } - // Emit the header - if(this.info.records === 0){ - this.bom(); - const err = this.headers(); - if(err) return err; - } - // Emit and stringify the record if an object or an array - try{ - this.emit('record', chunk, this.info.records); - }catch(err){ - return err; - } - // Convert the record into a string - let err, chunk_string; - if(this.options.eof){ - [err, chunk_string] = this.stringify(chunk); - if(err) return err; - if(chunk_string === undefined){ - return; - }else { - chunk_string = chunk_string + this.options.record_delimiter; + // Emit the header + if(this.info.records === 0){ + this.bom(push); + const err = this.headers(push); + if(err) return err; } - }else { - [err, chunk_string] = this.stringify(chunk); - if(err) return err; - if(chunk_string === undefined){ - return; - }else { - if(this.options.header || this.info.records){ - chunk_string = this.options.record_delimiter + chunk_string; + // Emit and stringify the record if an object or an array + try{ + // this.emit('record', chunk, this.info.records); + if(this.options.on_record){ + this.options.on_record(chunk, this.info.records); } + }catch(err){ + return err; } - } - // Emit the csv - this.info.records++; - this.push(chunk_string); - } - stringify(chunk, chunkIsHeader=false){ - if(typeof chunk !== 'object'){ - return [undefined, chunk]; - } - const {columns} = this.options; - const record = []; - // Record is an array - if(Array.isArray(chunk)){ - // We are getting an array but the user has specified output columns. In - // this case, we respect the columns indexes - if(columns){ - chunk.splice(columns.length); - } - // Cast record elements - for(let i=0; i= 0; - const containsQuote = (quote !== '') && value.indexOf(quote) >= 0; - const containsEscape = value.indexOf(escape) >= 0 && (escape !== quote); - const containsRecordDelimiter = value.indexOf(record_delimiter) >= 0; - const quotedString = quoted_string && typeof field === 'string'; - let quotedMatch = quoted_match && quoted_match.filter(quoted_match => { - if(typeof quoted_match === 'string'){ - return value.indexOf(quoted_match) !== -1; - }else { - return quoted_match.test(value); + } + let csvrecord = ''; + for(let i=0; i 0; - const shouldQuote = containsQuote === true || containsdelimiter || containsRecordDelimiter || quoted || quotedString || quotedMatch; - if(shouldQuote === true && containsEscape === true){ - const regexp = escape === '\\' - ? new RegExp(escape + escape, 'g') - : new RegExp(escape, 'g'); - value = value.replace(regexp, escape + escape); + options = {...this.options, ...options}; + [err, options] = normalize_options(options); + if(err !== undefined){ + return [err]; + } + }else if(value === undefined || value === null){ + options = this.options; + }else { + return [Error(`Invalid Casting Value: returned value must return a string, an object, null or undefined, got ${JSON.stringify(value)}`)]; } - if(containsQuote === true){ - const regexp = new RegExp(quote,'g'); - value = value.replace(regexp, escape + quote); + const {delimiter, escape, quote, quoted, quoted_empty, quoted_string, quoted_match, record_delimiter} = options; + if(value){ + if(typeof value !== 'string'){ + return [Error(`Formatter must return a string, null or undefined, got ${JSON.stringify(value)}`)]; + } + const containsdelimiter = delimiter.length && value.indexOf(delimiter) >= 0; + const containsQuote = (quote !== '') && value.indexOf(quote) >= 0; + const containsEscape = value.indexOf(escape) >= 0 && (escape !== quote); + const containsRecordDelimiter = value.indexOf(record_delimiter) >= 0; + const quotedString = quoted_string && typeof field === 'string'; + let quotedMatch = quoted_match && quoted_match.filter(quoted_match => { + if(typeof quoted_match === 'string'){ + return value.indexOf(quoted_match) !== -1; + }else { + return quoted_match.test(value); + } + }); + quotedMatch = quotedMatch && quotedMatch.length > 0; + const shouldQuote = containsQuote === true || containsdelimiter || containsRecordDelimiter || quoted || quotedString || quotedMatch; + if(shouldQuote === true && containsEscape === true){ + const regexp = escape === '\\' + ? new RegExp(escape + escape, 'g') + : new RegExp(escape, 'g'); + value = value.replace(regexp, escape + escape); + } + if(containsQuote === true){ + const regexp = new RegExp(quote,'g'); + value = value.replace(regexp, escape + quote); + } + if(shouldQuote === true){ + value = quote + value + quote; + } + csvrecord += value; + }else if(quoted_empty === true || (field === '' && quoted_string === true && quoted_empty !== false)){ + csvrecord += quote + quote; } - if(shouldQuote === true){ - value = quote + value + quote; + if(i !== record.length - 1){ + csvrecord += delimiter; } - csvrecord += value; - }else if(quoted_empty === true || (field === '' && quoted_string === true && quoted_empty !== false)){ - csvrecord += quote + quote; } - if(i !== record.length - 1){ - csvrecord += delimiter; + return [undefined, csvrecord]; + }, + bom: function(push){ + if(this.options.bom !== true){ + return; } - } - return [undefined, csvrecord]; - } - bom(){ - if(this.options.bom !== true){ - return; - } - this.push(bom_utf8); - } - headers(){ - if(this.options.header === false){ - return; - } - if(this.options.columns === undefined){ - return; - } - let err; - let headers = this.options.columns.map(column => column.header); - if(this.options.eof){ - [err, headers] = this.stringify(headers, true); - headers += this.options.record_delimiter; - }else { - [err, headers] = this.stringify(headers); - } - if(err) return err; - this.push(headers); - } - __cast(value, context){ - const type = typeof value; - try{ - if(type === 'string'){ // Fine for 99% of the cases - return [undefined, this.options.cast.string(value, context)]; - }else if(type === 'bigint'){ - return [undefined, this.options.cast.bigint(value, context)]; - }else if(type === 'number'){ - return [undefined, this.options.cast.number(value, context)]; - }else if(type === 'boolean'){ - return [undefined, this.options.cast.boolean(value, context)]; - }else if(value instanceof Date){ - return [undefined, this.options.cast.date(value, context)]; - }else if(type === 'object' && value !== null){ - return [undefined, this.options.cast.object(value, context)]; - }else { - return [undefined, value, value]; + push(bom_utf8); + }, + headers: function(push){ + if(this.options.header === false){ + return; } - }catch(err){ - return [err]; - } - } - normalize_columns(columns){ - if(columns === undefined || columns === null){ - return []; - } - if(typeof columns !== 'object'){ - return [Error('Invalid option "columns": expect an array or an object')]; - } - if(!Array.isArray(columns)){ - const newcolumns = []; - for(const k in columns){ - newcolumns.push({ - key: k, - header: columns[k] - }); + if(this.options.columns === undefined){ + return; } - columns = newcolumns; - }else { - const newcolumns = []; - for(const column of columns){ - if(typeof column === 'string'){ - newcolumns.push({ - key: column, - header: column - }); - }else if(typeof column === 'object' && column !== undefined && !Array.isArray(column)){ - if(!column.key){ - return [Error('Invalid column definition: property "key" is required')]; - } - if(column.header === undefined){ - column.header = column.key; - } - newcolumns.push(column); + let err; + let headers = this.options.columns.map(column => column.header); + if(this.options.eof){ + [err, headers] = this.stringify(headers, true); + headers += this.options.record_delimiter; + }else { + [err, headers] = this.stringify(headers); + } + if(err) return err; + push(headers); + }, + __cast: function(value, context){ + const type = typeof value; + try{ + if(type === 'string'){ // Fine for 99% of the cases + return [undefined, this.options.cast.string(value, context)]; + }else if(type === 'bigint'){ + return [undefined, this.options.cast.bigint(value, context)]; + }else if(type === 'number'){ + return [undefined, this.options.cast.number(value, context)]; + }else if(type === 'boolean'){ + return [undefined, this.options.cast.boolean(value, context)]; + }else if(value instanceof Date){ + return [undefined, this.options.cast.date(value, context)]; + }else if(type === 'object' && value !== null){ + return [undefined, this.options.cast.object(value, context)]; }else { - return [Error('Invalid column definition: expect a string or an object')]; + return [undefined, value, value]; } + }catch(err){ + return [err]; } - columns = newcolumns; } - return [undefined, columns]; - } -} + }; +}; -const stringify = function(records, options={}){ +const stringify = function(records, opts={}){ const data = []; - const stringifier = new Stringifier(options); - stringifier.push = function(record){ - if(record === null){ - return; - } - data.push(record.toString()); + const [err, options] = normalize_options(opts); + if(err !== undefined) throw err; + const state = { + stop: false }; + // Information + const info = { + records: 0 + }; + const api = stringifier(options, state, info); + // stringifier.push = function(record){ + // if(record === null){ + // return; + // } + // data.push(record.toString()); + // }; for(const record of records){ - const err = stringifier.__transform(record, null); + const err = api.__transform(record, function(record){ + data.push(record); + }); if(err !== undefined) throw err; } return data.join(''); diff --git a/packages/csv-stringify/dist/iife/index.js b/packages/csv-stringify/dist/iife/index.js index 3bbd95e9b..8331c85ed 100644 --- a/packages/csv-stringify/dist/iife/index.js +++ b/packages/csv-stringify/dist/iife/index.js @@ -1,5575 +1,5598 @@ var csv_stringify = (function (exports) { - 'use strict'; - - var global$1 = (typeof global !== "undefined" ? global : - typeof self !== "undefined" ? self : - typeof window !== "undefined" ? window : {}); - - var lookup = []; - var revLookup = []; - var Arr = typeof Uint8Array !== 'undefined' ? Uint8Array : Array; - var inited = false; - function init () { - inited = true; - var code = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'; - for (var i = 0, len = code.length; i < len; ++i) { - lookup[i] = code[i]; - revLookup[code.charCodeAt(i)] = i; - } - - revLookup['-'.charCodeAt(0)] = 62; - revLookup['_'.charCodeAt(0)] = 63; - } - - function toByteArray (b64) { - if (!inited) { - init(); - } - var i, j, l, tmp, placeHolders, arr; - var len = b64.length; - - if (len % 4 > 0) { - throw new Error('Invalid string. Length must be a multiple of 4') - } - - // the number of equal signs (place holders) - // if there are two placeholders, than the two characters before it - // represent one byte - // if there is only one, then the three characters before it represent 2 bytes - // this is just a cheap hack to not do indexOf twice - placeHolders = b64[len - 2] === '=' ? 2 : b64[len - 1] === '=' ? 1 : 0; - - // base64 is 4/3 + up to two characters of the original data - arr = new Arr(len * 3 / 4 - placeHolders); - - // if there are placeholders, only get up to the last complete 4 chars - l = placeHolders > 0 ? len - 4 : len; - - var L = 0; - - for (i = 0, j = 0; i < l; i += 4, j += 3) { - tmp = (revLookup[b64.charCodeAt(i)] << 18) | (revLookup[b64.charCodeAt(i + 1)] << 12) | (revLookup[b64.charCodeAt(i + 2)] << 6) | revLookup[b64.charCodeAt(i + 3)]; - arr[L++] = (tmp >> 16) & 0xFF; - arr[L++] = (tmp >> 8) & 0xFF; - arr[L++] = tmp & 0xFF; - } - - if (placeHolders === 2) { - tmp = (revLookup[b64.charCodeAt(i)] << 2) | (revLookup[b64.charCodeAt(i + 1)] >> 4); - arr[L++] = tmp & 0xFF; - } else if (placeHolders === 1) { - tmp = (revLookup[b64.charCodeAt(i)] << 10) | (revLookup[b64.charCodeAt(i + 1)] << 4) | (revLookup[b64.charCodeAt(i + 2)] >> 2); - arr[L++] = (tmp >> 8) & 0xFF; - arr[L++] = tmp & 0xFF; - } - - return arr - } - - function tripletToBase64 (num) { - return lookup[num >> 18 & 0x3F] + lookup[num >> 12 & 0x3F] + lookup[num >> 6 & 0x3F] + lookup[num & 0x3F] - } - - function encodeChunk (uint8, start, end) { - var tmp; - var output = []; - for (var i = start; i < end; i += 3) { - tmp = (uint8[i] << 16) + (uint8[i + 1] << 8) + (uint8[i + 2]); - output.push(tripletToBase64(tmp)); - } - return output.join('') - } - - function fromByteArray (uint8) { - if (!inited) { - init(); - } - var tmp; - var len = uint8.length; - var extraBytes = len % 3; // if we have 1 byte left, pad 2 bytes - var output = ''; - var parts = []; - var maxChunkLength = 16383; // must be multiple of 3 - - // go through the array every three bytes, we'll deal with trailing stuff later - for (var i = 0, len2 = len - extraBytes; i < len2; i += maxChunkLength) { - parts.push(encodeChunk(uint8, i, (i + maxChunkLength) > len2 ? len2 : (i + maxChunkLength))); - } - - // pad the end with zeros, but make sure to not forget the extra bytes - if (extraBytes === 1) { - tmp = uint8[len - 1]; - output += lookup[tmp >> 2]; - output += lookup[(tmp << 4) & 0x3F]; - output += '=='; - } else if (extraBytes === 2) { - tmp = (uint8[len - 2] << 8) + (uint8[len - 1]); - output += lookup[tmp >> 10]; - output += lookup[(tmp >> 4) & 0x3F]; - output += lookup[(tmp << 2) & 0x3F]; - output += '='; - } - - parts.push(output); - - return parts.join('') - } - - function read (buffer, offset, isLE, mLen, nBytes) { - var e, m; - var eLen = nBytes * 8 - mLen - 1; - var eMax = (1 << eLen) - 1; - var eBias = eMax >> 1; - var nBits = -7; - var i = isLE ? (nBytes - 1) : 0; - var d = isLE ? -1 : 1; - var s = buffer[offset + i]; - - i += d; - - e = s & ((1 << (-nBits)) - 1); - s >>= (-nBits); - nBits += eLen; - for (; nBits > 0; e = e * 256 + buffer[offset + i], i += d, nBits -= 8) {} - - m = e & ((1 << (-nBits)) - 1); - e >>= (-nBits); - nBits += mLen; - for (; nBits > 0; m = m * 256 + buffer[offset + i], i += d, nBits -= 8) {} - - if (e === 0) { - e = 1 - eBias; - } else if (e === eMax) { - return m ? NaN : ((s ? -1 : 1) * Infinity) - } else { - m = m + Math.pow(2, mLen); - e = e - eBias; - } - return (s ? -1 : 1) * m * Math.pow(2, e - mLen) - } - - function write (buffer, value, offset, isLE, mLen, nBytes) { - var e, m, c; - var eLen = nBytes * 8 - mLen - 1; - var eMax = (1 << eLen) - 1; - var eBias = eMax >> 1; - var rt = (mLen === 23 ? Math.pow(2, -24) - Math.pow(2, -77) : 0); - var i = isLE ? 0 : (nBytes - 1); - var d = isLE ? 1 : -1; - var s = value < 0 || (value === 0 && 1 / value < 0) ? 1 : 0; - - value = Math.abs(value); - - if (isNaN(value) || value === Infinity) { - m = isNaN(value) ? 1 : 0; - e = eMax; - } else { - e = Math.floor(Math.log(value) / Math.LN2); - if (value * (c = Math.pow(2, -e)) < 1) { - e--; - c *= 2; - } - if (e + eBias >= 1) { - value += rt / c; - } else { - value += rt * Math.pow(2, 1 - eBias); - } - if (value * c >= 2) { - e++; - c /= 2; - } - - if (e + eBias >= eMax) { - m = 0; - e = eMax; - } else if (e + eBias >= 1) { - m = (value * c - 1) * Math.pow(2, mLen); - e = e + eBias; - } else { - m = value * Math.pow(2, eBias - 1) * Math.pow(2, mLen); - e = 0; - } - } - - for (; mLen >= 8; buffer[offset + i] = m & 0xff, i += d, m /= 256, mLen -= 8) {} - - e = (e << mLen) | m; - eLen += mLen; - for (; eLen > 0; buffer[offset + i] = e & 0xff, i += d, e /= 256, eLen -= 8) {} - - buffer[offset + i - d] |= s * 128; - } - - var toString = {}.toString; - - var isArray$1 = Array.isArray || function (arr) { - return toString.call(arr) == '[object Array]'; - }; - - var INSPECT_MAX_BYTES = 50; - - /** - * If `Buffer.TYPED_ARRAY_SUPPORT`: - * === true Use Uint8Array implementation (fastest) - * === false Use Object implementation (most compatible, even IE6) - * - * Browsers that support typed arrays are IE 10+, Firefox 4+, Chrome 7+, Safari 5.1+, - * Opera 11.6+, iOS 4.2+. - * - * Due to various browser bugs, sometimes the Object implementation will be used even - * when the browser supports typed arrays. - * - * Note: - * - * - Firefox 4-29 lacks support for adding new properties to `Uint8Array` instances, - * See: https://bugzilla.mozilla.org/show_bug.cgi?id=695438. - * - * - Chrome 9-10 is missing the `TypedArray.prototype.subarray` function. - * - * - IE10 has a broken `TypedArray.prototype.subarray` function which returns arrays of - * incorrect length in some situations. - - * We detect these buggy browsers and set `Buffer.TYPED_ARRAY_SUPPORT` to `false` so they - * get the Object implementation, which is slower but behaves correctly. - */ - Buffer.TYPED_ARRAY_SUPPORT = global$1.TYPED_ARRAY_SUPPORT !== undefined - ? global$1.TYPED_ARRAY_SUPPORT - : true; - - function kMaxLength () { - return Buffer.TYPED_ARRAY_SUPPORT - ? 0x7fffffff - : 0x3fffffff - } - - function createBuffer (that, length) { - if (kMaxLength() < length) { - throw new RangeError('Invalid typed array length') - } - if (Buffer.TYPED_ARRAY_SUPPORT) { - // Return an augmented `Uint8Array` instance, for best performance - that = new Uint8Array(length); - that.__proto__ = Buffer.prototype; - } else { - // Fallback: Return an object instance of the Buffer class - if (that === null) { - that = new Buffer(length); - } - that.length = length; - } - - return that - } - - /** - * The Buffer constructor returns instances of `Uint8Array` that have their - * prototype changed to `Buffer.prototype`. Furthermore, `Buffer` is a subclass of - * `Uint8Array`, so the returned instances will have all the node `Buffer` methods - * and the `Uint8Array` methods. Square bracket notation works as expected -- it - * returns a single octet. - * - * The `Uint8Array` prototype remains unmodified. - */ - - function Buffer (arg, encodingOrOffset, length) { - if (!Buffer.TYPED_ARRAY_SUPPORT && !(this instanceof Buffer)) { - return new Buffer(arg, encodingOrOffset, length) - } - - // Common case. - if (typeof arg === 'number') { - if (typeof encodingOrOffset === 'string') { - throw new Error( - 'If encoding is specified then the first argument must be a string' - ) - } - return allocUnsafe(this, arg) - } - return from(this, arg, encodingOrOffset, length) - } - - Buffer.poolSize = 8192; // not used by this implementation - - // TODO: Legacy, not needed anymore. Remove in next major version. - Buffer._augment = function (arr) { - arr.__proto__ = Buffer.prototype; - return arr - }; - - function from (that, value, encodingOrOffset, length) { - if (typeof value === 'number') { - throw new TypeError('"value" argument must not be a number') - } - - if (typeof ArrayBuffer !== 'undefined' && value instanceof ArrayBuffer) { - return fromArrayBuffer(that, value, encodingOrOffset, length) - } - - if (typeof value === 'string') { - return fromString(that, value, encodingOrOffset) - } - - return fromObject(that, value) - } - - /** - * Functionally equivalent to Buffer(arg, encoding) but throws a TypeError - * if value is a number. - * Buffer.from(str[, encoding]) - * Buffer.from(array) - * Buffer.from(buffer) - * Buffer.from(arrayBuffer[, byteOffset[, length]]) - **/ - Buffer.from = function (value, encodingOrOffset, length) { - return from(null, value, encodingOrOffset, length) - }; - - if (Buffer.TYPED_ARRAY_SUPPORT) { - Buffer.prototype.__proto__ = Uint8Array.prototype; - Buffer.__proto__ = Uint8Array; - } - - function assertSize (size) { - if (typeof size !== 'number') { - throw new TypeError('"size" argument must be a number') - } else if (size < 0) { - throw new RangeError('"size" argument must not be negative') - } - } - - function alloc (that, size, fill, encoding) { - assertSize(size); - if (size <= 0) { - return createBuffer(that, size) - } - if (fill !== undefined) { - // Only pay attention to encoding if it's a string. This - // prevents accidentally sending in a number that would - // be interpretted as a start offset. - return typeof encoding === 'string' - ? createBuffer(that, size).fill(fill, encoding) - : createBuffer(that, size).fill(fill) - } - return createBuffer(that, size) - } - - /** - * Creates a new filled Buffer instance. - * alloc(size[, fill[, encoding]]) - **/ - Buffer.alloc = function (size, fill, encoding) { - return alloc(null, size, fill, encoding) - }; - - function allocUnsafe (that, size) { - assertSize(size); - that = createBuffer(that, size < 0 ? 0 : checked(size) | 0); - if (!Buffer.TYPED_ARRAY_SUPPORT) { - for (var i = 0; i < size; ++i) { - that[i] = 0; - } - } - return that - } - - /** - * Equivalent to Buffer(num), by default creates a non-zero-filled Buffer instance. - * */ - Buffer.allocUnsafe = function (size) { - return allocUnsafe(null, size) - }; - /** - * Equivalent to SlowBuffer(num), by default creates a non-zero-filled Buffer instance. - */ - Buffer.allocUnsafeSlow = function (size) { - return allocUnsafe(null, size) - }; - - function fromString (that, string, encoding) { - if (typeof encoding !== 'string' || encoding === '') { - encoding = 'utf8'; - } - - if (!Buffer.isEncoding(encoding)) { - throw new TypeError('"encoding" must be a valid string encoding') - } - - var length = byteLength(string, encoding) | 0; - that = createBuffer(that, length); - - var actual = that.write(string, encoding); - - if (actual !== length) { - // Writing a hex string, for example, that contains invalid characters will - // cause everything after the first invalid character to be ignored. (e.g. - // 'abxxcd' will be treated as 'ab') - that = that.slice(0, actual); - } - - return that - } - - function fromArrayLike (that, array) { - var length = array.length < 0 ? 0 : checked(array.length) | 0; - that = createBuffer(that, length); - for (var i = 0; i < length; i += 1) { - that[i] = array[i] & 255; - } - return that - } - - function fromArrayBuffer (that, array, byteOffset, length) { - array.byteLength; // this throws if `array` is not a valid ArrayBuffer - - if (byteOffset < 0 || array.byteLength < byteOffset) { - throw new RangeError('\'offset\' is out of bounds') - } - - if (array.byteLength < byteOffset + (length || 0)) { - throw new RangeError('\'length\' is out of bounds') - } - - if (byteOffset === undefined && length === undefined) { - array = new Uint8Array(array); - } else if (length === undefined) { - array = new Uint8Array(array, byteOffset); - } else { - array = new Uint8Array(array, byteOffset, length); - } - - if (Buffer.TYPED_ARRAY_SUPPORT) { - // Return an augmented `Uint8Array` instance, for best performance - that = array; - that.__proto__ = Buffer.prototype; - } else { - // Fallback: Return an object instance of the Buffer class - that = fromArrayLike(that, array); - } - return that - } - - function fromObject (that, obj) { - if (internalIsBuffer(obj)) { - var len = checked(obj.length) | 0; - that = createBuffer(that, len); - - if (that.length === 0) { - return that - } - - obj.copy(that, 0, 0, len); - return that - } - - if (obj) { - if ((typeof ArrayBuffer !== 'undefined' && - obj.buffer instanceof ArrayBuffer) || 'length' in obj) { - if (typeof obj.length !== 'number' || isnan(obj.length)) { - return createBuffer(that, 0) - } - return fromArrayLike(that, obj) - } - - if (obj.type === 'Buffer' && isArray$1(obj.data)) { - return fromArrayLike(that, obj.data) - } - } - - throw new TypeError('First argument must be a string, Buffer, ArrayBuffer, Array, or array-like object.') - } - - function checked (length) { - // Note: cannot use `length < kMaxLength()` here because that fails when - // length is NaN (which is otherwise coerced to zero.) - if (length >= kMaxLength()) { - throw new RangeError('Attempt to allocate Buffer larger than maximum ' + - 'size: 0x' + kMaxLength().toString(16) + ' bytes') - } - return length | 0 - } - Buffer.isBuffer = isBuffer; - function internalIsBuffer (b) { - return !!(b != null && b._isBuffer) - } - - Buffer.compare = function compare (a, b) { - if (!internalIsBuffer(a) || !internalIsBuffer(b)) { - throw new TypeError('Arguments must be Buffers') - } - - if (a === b) return 0 - - var x = a.length; - var y = b.length; - - for (var i = 0, len = Math.min(x, y); i < len; ++i) { - if (a[i] !== b[i]) { - x = a[i]; - y = b[i]; - break - } - } - - if (x < y) return -1 - if (y < x) return 1 - return 0 - }; - - Buffer.isEncoding = function isEncoding (encoding) { - switch (String(encoding).toLowerCase()) { - case 'hex': - case 'utf8': - case 'utf-8': - case 'ascii': - case 'latin1': - case 'binary': - case 'base64': - case 'ucs2': - case 'ucs-2': - case 'utf16le': - case 'utf-16le': - return true - default: - return false - } - }; - - Buffer.concat = function concat (list, length) { - if (!isArray$1(list)) { - throw new TypeError('"list" argument must be an Array of Buffers') - } - - if (list.length === 0) { - return Buffer.alloc(0) - } - - var i; - if (length === undefined) { - length = 0; - for (i = 0; i < list.length; ++i) { - length += list[i].length; - } - } - - var buffer = Buffer.allocUnsafe(length); - var pos = 0; - for (i = 0; i < list.length; ++i) { - var buf = list[i]; - if (!internalIsBuffer(buf)) { - throw new TypeError('"list" argument must be an Array of Buffers') - } - buf.copy(buffer, pos); - pos += buf.length; - } - return buffer - }; - - function byteLength (string, encoding) { - if (internalIsBuffer(string)) { - return string.length - } - if (typeof ArrayBuffer !== 'undefined' && typeof ArrayBuffer.isView === 'function' && - (ArrayBuffer.isView(string) || string instanceof ArrayBuffer)) { - return string.byteLength - } - if (typeof string !== 'string') { - string = '' + string; - } - - var len = string.length; - if (len === 0) return 0 - - // Use a for loop to avoid recursion - var loweredCase = false; - for (;;) { - switch (encoding) { - case 'ascii': - case 'latin1': - case 'binary': - return len - case 'utf8': - case 'utf-8': - case undefined: - return utf8ToBytes(string).length - case 'ucs2': - case 'ucs-2': - case 'utf16le': - case 'utf-16le': - return len * 2 - case 'hex': - return len >>> 1 - case 'base64': - return base64ToBytes(string).length - default: - if (loweredCase) return utf8ToBytes(string).length // assume utf8 - encoding = ('' + encoding).toLowerCase(); - loweredCase = true; - } - } - } - Buffer.byteLength = byteLength; - - function slowToString (encoding, start, end) { - var loweredCase = false; - - // No need to verify that "this.length <= MAX_UINT32" since it's a read-only - // property of a typed array. - - // This behaves neither like String nor Uint8Array in that we set start/end - // to their upper/lower bounds if the value passed is out of range. - // undefined is handled specially as per ECMA-262 6th Edition, - // Section 13.3.3.7 Runtime Semantics: KeyedBindingInitialization. - if (start === undefined || start < 0) { - start = 0; - } - // Return early if start > this.length. Done here to prevent potential uint32 - // coercion fail below. - if (start > this.length) { - return '' - } - - if (end === undefined || end > this.length) { - end = this.length; - } - - if (end <= 0) { - return '' - } - - // Force coersion to uint32. This will also coerce falsey/NaN values to 0. - end >>>= 0; - start >>>= 0; - - if (end <= start) { - return '' - } - - if (!encoding) encoding = 'utf8'; - - while (true) { - switch (encoding) { - case 'hex': - return hexSlice(this, start, end) - - case 'utf8': - case 'utf-8': - return utf8Slice(this, start, end) - - case 'ascii': - return asciiSlice(this, start, end) - - case 'latin1': - case 'binary': - return latin1Slice(this, start, end) - - case 'base64': - return base64Slice(this, start, end) - - case 'ucs2': - case 'ucs-2': - case 'utf16le': - case 'utf-16le': - return utf16leSlice(this, start, end) - - default: - if (loweredCase) throw new TypeError('Unknown encoding: ' + encoding) - encoding = (encoding + '').toLowerCase(); - loweredCase = true; - } - } - } - - // The property is used by `Buffer.isBuffer` and `is-buffer` (in Safari 5-7) to detect - // Buffer instances. - Buffer.prototype._isBuffer = true; - - function swap (b, n, m) { - var i = b[n]; - b[n] = b[m]; - b[m] = i; - } - - Buffer.prototype.swap16 = function swap16 () { - var len = this.length; - if (len % 2 !== 0) { - throw new RangeError('Buffer size must be a multiple of 16-bits') - } - for (var i = 0; i < len; i += 2) { - swap(this, i, i + 1); - } - return this - }; - - Buffer.prototype.swap32 = function swap32 () { - var len = this.length; - if (len % 4 !== 0) { - throw new RangeError('Buffer size must be a multiple of 32-bits') - } - for (var i = 0; i < len; i += 4) { - swap(this, i, i + 3); - swap(this, i + 1, i + 2); - } - return this - }; - - Buffer.prototype.swap64 = function swap64 () { - var len = this.length; - if (len % 8 !== 0) { - throw new RangeError('Buffer size must be a multiple of 64-bits') - } - for (var i = 0; i < len; i += 8) { - swap(this, i, i + 7); - swap(this, i + 1, i + 6); - swap(this, i + 2, i + 5); - swap(this, i + 3, i + 4); - } - return this - }; - - Buffer.prototype.toString = function toString () { - var length = this.length | 0; - if (length === 0) return '' - if (arguments.length === 0) return utf8Slice(this, 0, length) - return slowToString.apply(this, arguments) - }; - - Buffer.prototype.equals = function equals (b) { - if (!internalIsBuffer(b)) throw new TypeError('Argument must be a Buffer') - if (this === b) return true - return Buffer.compare(this, b) === 0 - }; - - Buffer.prototype.inspect = function inspect () { - var str = ''; - var max = INSPECT_MAX_BYTES; - if (this.length > 0) { - str = this.toString('hex', 0, max).match(/.{2}/g).join(' '); - if (this.length > max) str += ' ... '; - } - return '' - }; - - Buffer.prototype.compare = function compare (target, start, end, thisStart, thisEnd) { - if (!internalIsBuffer(target)) { - throw new TypeError('Argument must be a Buffer') - } - - if (start === undefined) { - start = 0; - } - if (end === undefined) { - end = target ? target.length : 0; - } - if (thisStart === undefined) { - thisStart = 0; - } - if (thisEnd === undefined) { - thisEnd = this.length; - } - - if (start < 0 || end > target.length || thisStart < 0 || thisEnd > this.length) { - throw new RangeError('out of range index') - } - - if (thisStart >= thisEnd && start >= end) { - return 0 - } - if (thisStart >= thisEnd) { - return -1 - } - if (start >= end) { - return 1 - } - - start >>>= 0; - end >>>= 0; - thisStart >>>= 0; - thisEnd >>>= 0; - - if (this === target) return 0 - - var x = thisEnd - thisStart; - var y = end - start; - var len = Math.min(x, y); - - var thisCopy = this.slice(thisStart, thisEnd); - var targetCopy = target.slice(start, end); - - for (var i = 0; i < len; ++i) { - if (thisCopy[i] !== targetCopy[i]) { - x = thisCopy[i]; - y = targetCopy[i]; - break - } - } - - if (x < y) return -1 - if (y < x) return 1 - return 0 - }; - - // Finds either the first index of `val` in `buffer` at offset >= `byteOffset`, - // OR the last index of `val` in `buffer` at offset <= `byteOffset`. - // - // Arguments: - // - buffer - a Buffer to search - // - val - a string, Buffer, or number - // - byteOffset - an index into `buffer`; will be clamped to an int32 - // - encoding - an optional encoding, relevant is val is a string - // - dir - true for indexOf, false for lastIndexOf - function bidirectionalIndexOf (buffer, val, byteOffset, encoding, dir) { - // Empty buffer means no match - if (buffer.length === 0) return -1 - - // Normalize byteOffset - if (typeof byteOffset === 'string') { - encoding = byteOffset; - byteOffset = 0; - } else if (byteOffset > 0x7fffffff) { - byteOffset = 0x7fffffff; - } else if (byteOffset < -0x80000000) { - byteOffset = -0x80000000; - } - byteOffset = +byteOffset; // Coerce to Number. - if (isNaN(byteOffset)) { - // byteOffset: it it's undefined, null, NaN, "foo", etc, search whole buffer - byteOffset = dir ? 0 : (buffer.length - 1); - } - - // Normalize byteOffset: negative offsets start from the end of the buffer - if (byteOffset < 0) byteOffset = buffer.length + byteOffset; - if (byteOffset >= buffer.length) { - if (dir) return -1 - else byteOffset = buffer.length - 1; - } else if (byteOffset < 0) { - if (dir) byteOffset = 0; - else return -1 - } - - // Normalize val - if (typeof val === 'string') { - val = Buffer.from(val, encoding); - } - - // Finally, search either indexOf (if dir is true) or lastIndexOf - if (internalIsBuffer(val)) { - // Special case: looking for empty string/buffer always fails - if (val.length === 0) { - return -1 - } - return arrayIndexOf(buffer, val, byteOffset, encoding, dir) - } else if (typeof val === 'number') { - val = val & 0xFF; // Search for a byte value [0-255] - if (Buffer.TYPED_ARRAY_SUPPORT && - typeof Uint8Array.prototype.indexOf === 'function') { - if (dir) { - return Uint8Array.prototype.indexOf.call(buffer, val, byteOffset) - } else { - return Uint8Array.prototype.lastIndexOf.call(buffer, val, byteOffset) - } - } - return arrayIndexOf(buffer, [ val ], byteOffset, encoding, dir) - } - - throw new TypeError('val must be string, number or Buffer') - } - - function arrayIndexOf (arr, val, byteOffset, encoding, dir) { - var indexSize = 1; - var arrLength = arr.length; - var valLength = val.length; - - if (encoding !== undefined) { - encoding = String(encoding).toLowerCase(); - if (encoding === 'ucs2' || encoding === 'ucs-2' || - encoding === 'utf16le' || encoding === 'utf-16le') { - if (arr.length < 2 || val.length < 2) { - return -1 - } - indexSize = 2; - arrLength /= 2; - valLength /= 2; - byteOffset /= 2; - } - } - - function read (buf, i) { - if (indexSize === 1) { - return buf[i] - } else { - return buf.readUInt16BE(i * indexSize) - } - } - - var i; - if (dir) { - var foundIndex = -1; - for (i = byteOffset; i < arrLength; i++) { - if (read(arr, i) === read(val, foundIndex === -1 ? 0 : i - foundIndex)) { - if (foundIndex === -1) foundIndex = i; - if (i - foundIndex + 1 === valLength) return foundIndex * indexSize - } else { - if (foundIndex !== -1) i -= i - foundIndex; - foundIndex = -1; - } - } - } else { - if (byteOffset + valLength > arrLength) byteOffset = arrLength - valLength; - for (i = byteOffset; i >= 0; i--) { - var found = true; - for (var j = 0; j < valLength; j++) { - if (read(arr, i + j) !== read(val, j)) { - found = false; - break - } - } - if (found) return i - } - } - - return -1 - } - - Buffer.prototype.includes = function includes (val, byteOffset, encoding) { - return this.indexOf(val, byteOffset, encoding) !== -1 - }; - - Buffer.prototype.indexOf = function indexOf (val, byteOffset, encoding) { - return bidirectionalIndexOf(this, val, byteOffset, encoding, true) - }; - - Buffer.prototype.lastIndexOf = function lastIndexOf (val, byteOffset, encoding) { - return bidirectionalIndexOf(this, val, byteOffset, encoding, false) - }; - - function hexWrite (buf, string, offset, length) { - offset = Number(offset) || 0; - var remaining = buf.length - offset; - if (!length) { - length = remaining; - } else { - length = Number(length); - if (length > remaining) { - length = remaining; - } - } - - // must be an even number of digits - var strLen = string.length; - if (strLen % 2 !== 0) throw new TypeError('Invalid hex string') - - if (length > strLen / 2) { - length = strLen / 2; - } - for (var i = 0; i < length; ++i) { - var parsed = parseInt(string.substr(i * 2, 2), 16); - if (isNaN(parsed)) return i - buf[offset + i] = parsed; - } - return i - } - - function utf8Write (buf, string, offset, length) { - return blitBuffer(utf8ToBytes(string, buf.length - offset), buf, offset, length) - } - - function asciiWrite (buf, string, offset, length) { - return blitBuffer(asciiToBytes(string), buf, offset, length) - } - - function latin1Write (buf, string, offset, length) { - return asciiWrite(buf, string, offset, length) - } - - function base64Write (buf, string, offset, length) { - return blitBuffer(base64ToBytes(string), buf, offset, length) - } - - function ucs2Write (buf, string, offset, length) { - return blitBuffer(utf16leToBytes(string, buf.length - offset), buf, offset, length) - } - - Buffer.prototype.write = function write (string, offset, length, encoding) { - // Buffer#write(string) - if (offset === undefined) { - encoding = 'utf8'; - length = this.length; - offset = 0; - // Buffer#write(string, encoding) - } else if (length === undefined && typeof offset === 'string') { - encoding = offset; - length = this.length; - offset = 0; - // Buffer#write(string, offset[, length][, encoding]) - } else if (isFinite(offset)) { - offset = offset | 0; - if (isFinite(length)) { - length = length | 0; - if (encoding === undefined) encoding = 'utf8'; - } else { - encoding = length; - length = undefined; - } - // legacy write(string, encoding, offset, length) - remove in v0.13 - } else { - throw new Error( - 'Buffer.write(string, encoding, offset[, length]) is no longer supported' - ) - } - - var remaining = this.length - offset; - if (length === undefined || length > remaining) length = remaining; - - if ((string.length > 0 && (length < 0 || offset < 0)) || offset > this.length) { - throw new RangeError('Attempt to write outside buffer bounds') - } - - if (!encoding) encoding = 'utf8'; - - var loweredCase = false; - for (;;) { - switch (encoding) { - case 'hex': - return hexWrite(this, string, offset, length) - - case 'utf8': - case 'utf-8': - return utf8Write(this, string, offset, length) - - case 'ascii': - return asciiWrite(this, string, offset, length) - - case 'latin1': - case 'binary': - return latin1Write(this, string, offset, length) - - case 'base64': - // Warning: maxLength not taken into account in base64Write - return base64Write(this, string, offset, length) - - case 'ucs2': - case 'ucs-2': - case 'utf16le': - case 'utf-16le': - return ucs2Write(this, string, offset, length) - - default: - if (loweredCase) throw new TypeError('Unknown encoding: ' + encoding) - encoding = ('' + encoding).toLowerCase(); - loweredCase = true; - } - } - }; - - Buffer.prototype.toJSON = function toJSON () { - return { - type: 'Buffer', - data: Array.prototype.slice.call(this._arr || this, 0) - } - }; - - function base64Slice (buf, start, end) { - if (start === 0 && end === buf.length) { - return fromByteArray(buf) - } else { - return fromByteArray(buf.slice(start, end)) - } - } - - function utf8Slice (buf, start, end) { - end = Math.min(buf.length, end); - var res = []; - - var i = start; - while (i < end) { - var firstByte = buf[i]; - var codePoint = null; - var bytesPerSequence = (firstByte > 0xEF) ? 4 - : (firstByte > 0xDF) ? 3 - : (firstByte > 0xBF) ? 2 - : 1; - - if (i + bytesPerSequence <= end) { - var secondByte, thirdByte, fourthByte, tempCodePoint; - - switch (bytesPerSequence) { - case 1: - if (firstByte < 0x80) { - codePoint = firstByte; - } - break - case 2: - secondByte = buf[i + 1]; - if ((secondByte & 0xC0) === 0x80) { - tempCodePoint = (firstByte & 0x1F) << 0x6 | (secondByte & 0x3F); - if (tempCodePoint > 0x7F) { - codePoint = tempCodePoint; - } - } - break - case 3: - secondByte = buf[i + 1]; - thirdByte = buf[i + 2]; - if ((secondByte & 0xC0) === 0x80 && (thirdByte & 0xC0) === 0x80) { - tempCodePoint = (firstByte & 0xF) << 0xC | (secondByte & 0x3F) << 0x6 | (thirdByte & 0x3F); - if (tempCodePoint > 0x7FF && (tempCodePoint < 0xD800 || tempCodePoint > 0xDFFF)) { - codePoint = tempCodePoint; - } - } - break - case 4: - secondByte = buf[i + 1]; - thirdByte = buf[i + 2]; - fourthByte = buf[i + 3]; - if ((secondByte & 0xC0) === 0x80 && (thirdByte & 0xC0) === 0x80 && (fourthByte & 0xC0) === 0x80) { - tempCodePoint = (firstByte & 0xF) << 0x12 | (secondByte & 0x3F) << 0xC | (thirdByte & 0x3F) << 0x6 | (fourthByte & 0x3F); - if (tempCodePoint > 0xFFFF && tempCodePoint < 0x110000) { - codePoint = tempCodePoint; - } - } - } - } - - if (codePoint === null) { - // we did not generate a valid codePoint so insert a - // replacement char (U+FFFD) and advance only 1 byte - codePoint = 0xFFFD; - bytesPerSequence = 1; - } else if (codePoint > 0xFFFF) { - // encode to utf16 (surrogate pair dance) - codePoint -= 0x10000; - res.push(codePoint >>> 10 & 0x3FF | 0xD800); - codePoint = 0xDC00 | codePoint & 0x3FF; - } - - res.push(codePoint); - i += bytesPerSequence; - } - - return decodeCodePointsArray(res) - } - - // Based on http://stackoverflow.com/a/22747272/680742, the browser with - // the lowest limit is Chrome, with 0x10000 args. - // We go 1 magnitude less, for safety - var MAX_ARGUMENTS_LENGTH = 0x1000; - - function decodeCodePointsArray (codePoints) { - var len = codePoints.length; - if (len <= MAX_ARGUMENTS_LENGTH) { - return String.fromCharCode.apply(String, codePoints) // avoid extra slice() - } - - // Decode in chunks to avoid "call stack size exceeded". - var res = ''; - var i = 0; - while (i < len) { - res += String.fromCharCode.apply( - String, - codePoints.slice(i, i += MAX_ARGUMENTS_LENGTH) - ); - } - return res - } - - function asciiSlice (buf, start, end) { - var ret = ''; - end = Math.min(buf.length, end); - - for (var i = start; i < end; ++i) { - ret += String.fromCharCode(buf[i] & 0x7F); - } - return ret - } - - function latin1Slice (buf, start, end) { - var ret = ''; - end = Math.min(buf.length, end); - - for (var i = start; i < end; ++i) { - ret += String.fromCharCode(buf[i]); - } - return ret - } - - function hexSlice (buf, start, end) { - var len = buf.length; - - if (!start || start < 0) start = 0; - if (!end || end < 0 || end > len) end = len; - - var out = ''; - for (var i = start; i < end; ++i) { - out += toHex(buf[i]); - } - return out - } - - function utf16leSlice (buf, start, end) { - var bytes = buf.slice(start, end); - var res = ''; - for (var i = 0; i < bytes.length; i += 2) { - res += String.fromCharCode(bytes[i] + bytes[i + 1] * 256); - } - return res - } - - Buffer.prototype.slice = function slice (start, end) { - var len = this.length; - start = ~~start; - end = end === undefined ? len : ~~end; - - if (start < 0) { - start += len; - if (start < 0) start = 0; - } else if (start > len) { - start = len; - } - - if (end < 0) { - end += len; - if (end < 0) end = 0; - } else if (end > len) { - end = len; - } - - if (end < start) end = start; - - var newBuf; - if (Buffer.TYPED_ARRAY_SUPPORT) { - newBuf = this.subarray(start, end); - newBuf.__proto__ = Buffer.prototype; - } else { - var sliceLen = end - start; - newBuf = new Buffer(sliceLen, undefined); - for (var i = 0; i < sliceLen; ++i) { - newBuf[i] = this[i + start]; - } - } - - return newBuf - }; - - /* - * Need to make sure that buffer isn't trying to write out of bounds. - */ - function checkOffset (offset, ext, length) { - if ((offset % 1) !== 0 || offset < 0) throw new RangeError('offset is not uint') - if (offset + ext > length) throw new RangeError('Trying to access beyond buffer length') - } - - Buffer.prototype.readUIntLE = function readUIntLE (offset, byteLength, noAssert) { - offset = offset | 0; - byteLength = byteLength | 0; - if (!noAssert) checkOffset(offset, byteLength, this.length); - - var val = this[offset]; - var mul = 1; - var i = 0; - while (++i < byteLength && (mul *= 0x100)) { - val += this[offset + i] * mul; - } - - return val - }; - - Buffer.prototype.readUIntBE = function readUIntBE (offset, byteLength, noAssert) { - offset = offset | 0; - byteLength = byteLength | 0; - if (!noAssert) { - checkOffset(offset, byteLength, this.length); - } - - var val = this[offset + --byteLength]; - var mul = 1; - while (byteLength > 0 && (mul *= 0x100)) { - val += this[offset + --byteLength] * mul; - } - - return val - }; - - Buffer.prototype.readUInt8 = function readUInt8 (offset, noAssert) { - if (!noAssert) checkOffset(offset, 1, this.length); - return this[offset] - }; - - Buffer.prototype.readUInt16LE = function readUInt16LE (offset, noAssert) { - if (!noAssert) checkOffset(offset, 2, this.length); - return this[offset] | (this[offset + 1] << 8) - }; - - Buffer.prototype.readUInt16BE = function readUInt16BE (offset, noAssert) { - if (!noAssert) checkOffset(offset, 2, this.length); - return (this[offset] << 8) | this[offset + 1] - }; - - Buffer.prototype.readUInt32LE = function readUInt32LE (offset, noAssert) { - if (!noAssert) checkOffset(offset, 4, this.length); - - return ((this[offset]) | - (this[offset + 1] << 8) | - (this[offset + 2] << 16)) + - (this[offset + 3] * 0x1000000) - }; - - Buffer.prototype.readUInt32BE = function readUInt32BE (offset, noAssert) { - if (!noAssert) checkOffset(offset, 4, this.length); - - return (this[offset] * 0x1000000) + - ((this[offset + 1] << 16) | - (this[offset + 2] << 8) | - this[offset + 3]) - }; - - Buffer.prototype.readIntLE = function readIntLE (offset, byteLength, noAssert) { - offset = offset | 0; - byteLength = byteLength | 0; - if (!noAssert) checkOffset(offset, byteLength, this.length); - - var val = this[offset]; - var mul = 1; - var i = 0; - while (++i < byteLength && (mul *= 0x100)) { - val += this[offset + i] * mul; - } - mul *= 0x80; - - if (val >= mul) val -= Math.pow(2, 8 * byteLength); - - return val - }; - - Buffer.prototype.readIntBE = function readIntBE (offset, byteLength, noAssert) { - offset = offset | 0; - byteLength = byteLength | 0; - if (!noAssert) checkOffset(offset, byteLength, this.length); - - var i = byteLength; - var mul = 1; - var val = this[offset + --i]; - while (i > 0 && (mul *= 0x100)) { - val += this[offset + --i] * mul; - } - mul *= 0x80; - - if (val >= mul) val -= Math.pow(2, 8 * byteLength); - - return val - }; - - Buffer.prototype.readInt8 = function readInt8 (offset, noAssert) { - if (!noAssert) checkOffset(offset, 1, this.length); - if (!(this[offset] & 0x80)) return (this[offset]) - return ((0xff - this[offset] + 1) * -1) - }; - - Buffer.prototype.readInt16LE = function readInt16LE (offset, noAssert) { - if (!noAssert) checkOffset(offset, 2, this.length); - var val = this[offset] | (this[offset + 1] << 8); - return (val & 0x8000) ? val | 0xFFFF0000 : val - }; - - Buffer.prototype.readInt16BE = function readInt16BE (offset, noAssert) { - if (!noAssert) checkOffset(offset, 2, this.length); - var val = this[offset + 1] | (this[offset] << 8); - return (val & 0x8000) ? val | 0xFFFF0000 : val - }; - - Buffer.prototype.readInt32LE = function readInt32LE (offset, noAssert) { - if (!noAssert) checkOffset(offset, 4, this.length); - - return (this[offset]) | - (this[offset + 1] << 8) | - (this[offset + 2] << 16) | - (this[offset + 3] << 24) - }; - - Buffer.prototype.readInt32BE = function readInt32BE (offset, noAssert) { - if (!noAssert) checkOffset(offset, 4, this.length); - - return (this[offset] << 24) | - (this[offset + 1] << 16) | - (this[offset + 2] << 8) | - (this[offset + 3]) - }; - - Buffer.prototype.readFloatLE = function readFloatLE (offset, noAssert) { - if (!noAssert) checkOffset(offset, 4, this.length); - return read(this, offset, true, 23, 4) - }; - - Buffer.prototype.readFloatBE = function readFloatBE (offset, noAssert) { - if (!noAssert) checkOffset(offset, 4, this.length); - return read(this, offset, false, 23, 4) - }; - - Buffer.prototype.readDoubleLE = function readDoubleLE (offset, noAssert) { - if (!noAssert) checkOffset(offset, 8, this.length); - return read(this, offset, true, 52, 8) - }; - - Buffer.prototype.readDoubleBE = function readDoubleBE (offset, noAssert) { - if (!noAssert) checkOffset(offset, 8, this.length); - return read(this, offset, false, 52, 8) - }; - - function checkInt (buf, value, offset, ext, max, min) { - if (!internalIsBuffer(buf)) throw new TypeError('"buffer" argument must be a Buffer instance') - if (value > max || value < min) throw new RangeError('"value" argument is out of bounds') - if (offset + ext > buf.length) throw new RangeError('Index out of range') - } - - Buffer.prototype.writeUIntLE = function writeUIntLE (value, offset, byteLength, noAssert) { - value = +value; - offset = offset | 0; - byteLength = byteLength | 0; - if (!noAssert) { - var maxBytes = Math.pow(2, 8 * byteLength) - 1; - checkInt(this, value, offset, byteLength, maxBytes, 0); - } - - var mul = 1; - var i = 0; - this[offset] = value & 0xFF; - while (++i < byteLength && (mul *= 0x100)) { - this[offset + i] = (value / mul) & 0xFF; - } - - return offset + byteLength - }; - - Buffer.prototype.writeUIntBE = function writeUIntBE (value, offset, byteLength, noAssert) { - value = +value; - offset = offset | 0; - byteLength = byteLength | 0; - if (!noAssert) { - var maxBytes = Math.pow(2, 8 * byteLength) - 1; - checkInt(this, value, offset, byteLength, maxBytes, 0); - } - - var i = byteLength - 1; - var mul = 1; - this[offset + i] = value & 0xFF; - while (--i >= 0 && (mul *= 0x100)) { - this[offset + i] = (value / mul) & 0xFF; - } - - return offset + byteLength - }; - - Buffer.prototype.writeUInt8 = function writeUInt8 (value, offset, noAssert) { - value = +value; - offset = offset | 0; - if (!noAssert) checkInt(this, value, offset, 1, 0xff, 0); - if (!Buffer.TYPED_ARRAY_SUPPORT) value = Math.floor(value); - this[offset] = (value & 0xff); - return offset + 1 - }; - - function objectWriteUInt16 (buf, value, offset, littleEndian) { - if (value < 0) value = 0xffff + value + 1; - for (var i = 0, j = Math.min(buf.length - offset, 2); i < j; ++i) { - buf[offset + i] = (value & (0xff << (8 * (littleEndian ? i : 1 - i)))) >>> - (littleEndian ? i : 1 - i) * 8; - } - } - - Buffer.prototype.writeUInt16LE = function writeUInt16LE (value, offset, noAssert) { - value = +value; - offset = offset | 0; - if (!noAssert) checkInt(this, value, offset, 2, 0xffff, 0); - if (Buffer.TYPED_ARRAY_SUPPORT) { - this[offset] = (value & 0xff); - this[offset + 1] = (value >>> 8); - } else { - objectWriteUInt16(this, value, offset, true); - } - return offset + 2 - }; - - Buffer.prototype.writeUInt16BE = function writeUInt16BE (value, offset, noAssert) { - value = +value; - offset = offset | 0; - if (!noAssert) checkInt(this, value, offset, 2, 0xffff, 0); - if (Buffer.TYPED_ARRAY_SUPPORT) { - this[offset] = (value >>> 8); - this[offset + 1] = (value & 0xff); - } else { - objectWriteUInt16(this, value, offset, false); - } - return offset + 2 - }; - - function objectWriteUInt32 (buf, value, offset, littleEndian) { - if (value < 0) value = 0xffffffff + value + 1; - for (var i = 0, j = Math.min(buf.length - offset, 4); i < j; ++i) { - buf[offset + i] = (value >>> (littleEndian ? i : 3 - i) * 8) & 0xff; - } - } - - Buffer.prototype.writeUInt32LE = function writeUInt32LE (value, offset, noAssert) { - value = +value; - offset = offset | 0; - if (!noAssert) checkInt(this, value, offset, 4, 0xffffffff, 0); - if (Buffer.TYPED_ARRAY_SUPPORT) { - this[offset + 3] = (value >>> 24); - this[offset + 2] = (value >>> 16); - this[offset + 1] = (value >>> 8); - this[offset] = (value & 0xff); - } else { - objectWriteUInt32(this, value, offset, true); - } - return offset + 4 - }; - - Buffer.prototype.writeUInt32BE = function writeUInt32BE (value, offset, noAssert) { - value = +value; - offset = offset | 0; - if (!noAssert) checkInt(this, value, offset, 4, 0xffffffff, 0); - if (Buffer.TYPED_ARRAY_SUPPORT) { - this[offset] = (value >>> 24); - this[offset + 1] = (value >>> 16); - this[offset + 2] = (value >>> 8); - this[offset + 3] = (value & 0xff); - } else { - objectWriteUInt32(this, value, offset, false); - } - return offset + 4 - }; - - Buffer.prototype.writeIntLE = function writeIntLE (value, offset, byteLength, noAssert) { - value = +value; - offset = offset | 0; - if (!noAssert) { - var limit = Math.pow(2, 8 * byteLength - 1); - - checkInt(this, value, offset, byteLength, limit - 1, -limit); - } - - var i = 0; - var mul = 1; - var sub = 0; - this[offset] = value & 0xFF; - while (++i < byteLength && (mul *= 0x100)) { - if (value < 0 && sub === 0 && this[offset + i - 1] !== 0) { - sub = 1; - } - this[offset + i] = ((value / mul) >> 0) - sub & 0xFF; - } - - return offset + byteLength - }; - - Buffer.prototype.writeIntBE = function writeIntBE (value, offset, byteLength, noAssert) { - value = +value; - offset = offset | 0; - if (!noAssert) { - var limit = Math.pow(2, 8 * byteLength - 1); - - checkInt(this, value, offset, byteLength, limit - 1, -limit); - } - - var i = byteLength - 1; - var mul = 1; - var sub = 0; - this[offset + i] = value & 0xFF; - while (--i >= 0 && (mul *= 0x100)) { - if (value < 0 && sub === 0 && this[offset + i + 1] !== 0) { - sub = 1; - } - this[offset + i] = ((value / mul) >> 0) - sub & 0xFF; - } - - return offset + byteLength - }; - - Buffer.prototype.writeInt8 = function writeInt8 (value, offset, noAssert) { - value = +value; - offset = offset | 0; - if (!noAssert) checkInt(this, value, offset, 1, 0x7f, -0x80); - if (!Buffer.TYPED_ARRAY_SUPPORT) value = Math.floor(value); - if (value < 0) value = 0xff + value + 1; - this[offset] = (value & 0xff); - return offset + 1 - }; - - Buffer.prototype.writeInt16LE = function writeInt16LE (value, offset, noAssert) { - value = +value; - offset = offset | 0; - if (!noAssert) checkInt(this, value, offset, 2, 0x7fff, -0x8000); - if (Buffer.TYPED_ARRAY_SUPPORT) { - this[offset] = (value & 0xff); - this[offset + 1] = (value >>> 8); - } else { - objectWriteUInt16(this, value, offset, true); - } - return offset + 2 - }; - - Buffer.prototype.writeInt16BE = function writeInt16BE (value, offset, noAssert) { - value = +value; - offset = offset | 0; - if (!noAssert) checkInt(this, value, offset, 2, 0x7fff, -0x8000); - if (Buffer.TYPED_ARRAY_SUPPORT) { - this[offset] = (value >>> 8); - this[offset + 1] = (value & 0xff); - } else { - objectWriteUInt16(this, value, offset, false); - } - return offset + 2 - }; - - Buffer.prototype.writeInt32LE = function writeInt32LE (value, offset, noAssert) { - value = +value; - offset = offset | 0; - if (!noAssert) checkInt(this, value, offset, 4, 0x7fffffff, -0x80000000); - if (Buffer.TYPED_ARRAY_SUPPORT) { - this[offset] = (value & 0xff); - this[offset + 1] = (value >>> 8); - this[offset + 2] = (value >>> 16); - this[offset + 3] = (value >>> 24); - } else { - objectWriteUInt32(this, value, offset, true); - } - return offset + 4 - }; - - Buffer.prototype.writeInt32BE = function writeInt32BE (value, offset, noAssert) { - value = +value; - offset = offset | 0; - if (!noAssert) checkInt(this, value, offset, 4, 0x7fffffff, -0x80000000); - if (value < 0) value = 0xffffffff + value + 1; - if (Buffer.TYPED_ARRAY_SUPPORT) { - this[offset] = (value >>> 24); - this[offset + 1] = (value >>> 16); - this[offset + 2] = (value >>> 8); - this[offset + 3] = (value & 0xff); - } else { - objectWriteUInt32(this, value, offset, false); - } - return offset + 4 - }; - - function checkIEEE754 (buf, value, offset, ext, max, min) { - if (offset + ext > buf.length) throw new RangeError('Index out of range') - if (offset < 0) throw new RangeError('Index out of range') - } - - function writeFloat (buf, value, offset, littleEndian, noAssert) { - if (!noAssert) { - checkIEEE754(buf, value, offset, 4); - } - write(buf, value, offset, littleEndian, 23, 4); - return offset + 4 - } - - Buffer.prototype.writeFloatLE = function writeFloatLE (value, offset, noAssert) { - return writeFloat(this, value, offset, true, noAssert) - }; - - Buffer.prototype.writeFloatBE = function writeFloatBE (value, offset, noAssert) { - return writeFloat(this, value, offset, false, noAssert) - }; - - function writeDouble (buf, value, offset, littleEndian, noAssert) { - if (!noAssert) { - checkIEEE754(buf, value, offset, 8); - } - write(buf, value, offset, littleEndian, 52, 8); - return offset + 8 - } - - Buffer.prototype.writeDoubleLE = function writeDoubleLE (value, offset, noAssert) { - return writeDouble(this, value, offset, true, noAssert) - }; - - Buffer.prototype.writeDoubleBE = function writeDoubleBE (value, offset, noAssert) { - return writeDouble(this, value, offset, false, noAssert) - }; - - // copy(targetBuffer, targetStart=0, sourceStart=0, sourceEnd=buffer.length) - Buffer.prototype.copy = function copy (target, targetStart, start, end) { - if (!start) start = 0; - if (!end && end !== 0) end = this.length; - if (targetStart >= target.length) targetStart = target.length; - if (!targetStart) targetStart = 0; - if (end > 0 && end < start) end = start; - - // Copy 0 bytes; we're done - if (end === start) return 0 - if (target.length === 0 || this.length === 0) return 0 - - // Fatal error conditions - if (targetStart < 0) { - throw new RangeError('targetStart out of bounds') - } - if (start < 0 || start >= this.length) throw new RangeError('sourceStart out of bounds') - if (end < 0) throw new RangeError('sourceEnd out of bounds') - - // Are we oob? - if (end > this.length) end = this.length; - if (target.length - targetStart < end - start) { - end = target.length - targetStart + start; - } - - var len = end - start; - var i; - - if (this === target && start < targetStart && targetStart < end) { - // descending copy from end - for (i = len - 1; i >= 0; --i) { - target[i + targetStart] = this[i + start]; - } - } else if (len < 1000 || !Buffer.TYPED_ARRAY_SUPPORT) { - // ascending copy from start - for (i = 0; i < len; ++i) { - target[i + targetStart] = this[i + start]; - } - } else { - Uint8Array.prototype.set.call( - target, - this.subarray(start, start + len), - targetStart - ); - } - - return len - }; - - // Usage: - // buffer.fill(number[, offset[, end]]) - // buffer.fill(buffer[, offset[, end]]) - // buffer.fill(string[, offset[, end]][, encoding]) - Buffer.prototype.fill = function fill (val, start, end, encoding) { - // Handle string cases: - if (typeof val === 'string') { - if (typeof start === 'string') { - encoding = start; - start = 0; - end = this.length; - } else if (typeof end === 'string') { - encoding = end; - end = this.length; - } - if (val.length === 1) { - var code = val.charCodeAt(0); - if (code < 256) { - val = code; - } - } - if (encoding !== undefined && typeof encoding !== 'string') { - throw new TypeError('encoding must be a string') - } - if (typeof encoding === 'string' && !Buffer.isEncoding(encoding)) { - throw new TypeError('Unknown encoding: ' + encoding) - } - } else if (typeof val === 'number') { - val = val & 255; - } - - // Invalid ranges are not set to a default, so can range check early. - if (start < 0 || this.length < start || this.length < end) { - throw new RangeError('Out of range index') - } - - if (end <= start) { - return this - } - - start = start >>> 0; - end = end === undefined ? this.length : end >>> 0; - - if (!val) val = 0; - - var i; - if (typeof val === 'number') { - for (i = start; i < end; ++i) { - this[i] = val; - } - } else { - var bytes = internalIsBuffer(val) - ? val - : utf8ToBytes(new Buffer(val, encoding).toString()); - var len = bytes.length; - for (i = 0; i < end - start; ++i) { - this[i + start] = bytes[i % len]; - } - } - - return this - }; - - // HELPER FUNCTIONS - // ================ - - var INVALID_BASE64_RE = /[^+\/0-9A-Za-z-_]/g; - - function base64clean (str) { - // Node strips out invalid characters like \n and \t from the string, base64-js does not - str = stringtrim(str).replace(INVALID_BASE64_RE, ''); - // Node converts strings with length < 2 to '' - if (str.length < 2) return '' - // Node allows for non-padded base64 strings (missing trailing ===), base64-js does not - while (str.length % 4 !== 0) { - str = str + '='; - } - return str - } - - function stringtrim (str) { - if (str.trim) return str.trim() - return str.replace(/^\s+|\s+$/g, '') - } - - function toHex (n) { - if (n < 16) return '0' + n.toString(16) - return n.toString(16) - } - - function utf8ToBytes (string, units) { - units = units || Infinity; - var codePoint; - var length = string.length; - var leadSurrogate = null; - var bytes = []; - - for (var i = 0; i < length; ++i) { - codePoint = string.charCodeAt(i); - - // is surrogate component - if (codePoint > 0xD7FF && codePoint < 0xE000) { - // last char was a lead - if (!leadSurrogate) { - // no lead yet - if (codePoint > 0xDBFF) { - // unexpected trail - if ((units -= 3) > -1) bytes.push(0xEF, 0xBF, 0xBD); - continue - } else if (i + 1 === length) { - // unpaired lead - if ((units -= 3) > -1) bytes.push(0xEF, 0xBF, 0xBD); - continue - } - - // valid lead - leadSurrogate = codePoint; - - continue - } - - // 2 leads in a row - if (codePoint < 0xDC00) { - if ((units -= 3) > -1) bytes.push(0xEF, 0xBF, 0xBD); - leadSurrogate = codePoint; - continue - } - - // valid surrogate pair - codePoint = (leadSurrogate - 0xD800 << 10 | codePoint - 0xDC00) + 0x10000; - } else if (leadSurrogate) { - // valid bmp char, but last char was a lead - if ((units -= 3) > -1) bytes.push(0xEF, 0xBF, 0xBD); - } - - leadSurrogate = null; - - // encode utf8 - if (codePoint < 0x80) { - if ((units -= 1) < 0) break - bytes.push(codePoint); - } else if (codePoint < 0x800) { - if ((units -= 2) < 0) break - bytes.push( - codePoint >> 0x6 | 0xC0, - codePoint & 0x3F | 0x80 - ); - } else if (codePoint < 0x10000) { - if ((units -= 3) < 0) break - bytes.push( - codePoint >> 0xC | 0xE0, - codePoint >> 0x6 & 0x3F | 0x80, - codePoint & 0x3F | 0x80 - ); - } else if (codePoint < 0x110000) { - if ((units -= 4) < 0) break - bytes.push( - codePoint >> 0x12 | 0xF0, - codePoint >> 0xC & 0x3F | 0x80, - codePoint >> 0x6 & 0x3F | 0x80, - codePoint & 0x3F | 0x80 - ); - } else { - throw new Error('Invalid code point') - } - } - - return bytes - } - - function asciiToBytes (str) { - var byteArray = []; - for (var i = 0; i < str.length; ++i) { - // Node's code seems to be doing this and not & 0x7F.. - byteArray.push(str.charCodeAt(i) & 0xFF); - } - return byteArray - } - - function utf16leToBytes (str, units) { - var c, hi, lo; - var byteArray = []; - for (var i = 0; i < str.length; ++i) { - if ((units -= 2) < 0) break - - c = str.charCodeAt(i); - hi = c >> 8; - lo = c % 256; - byteArray.push(lo); - byteArray.push(hi); - } - - return byteArray - } - - - function base64ToBytes (str) { - return toByteArray(base64clean(str)) - } - - function blitBuffer (src, dst, offset, length) { - for (var i = 0; i < length; ++i) { - if ((i + offset >= dst.length) || (i >= src.length)) break - dst[i + offset] = src[i]; - } - return i - } - - function isnan (val) { - return val !== val // eslint-disable-line no-self-compare - } - - - // the following is from is-buffer, also by Feross Aboukhadijeh and with same lisence - // The _isBuffer check is for Safari 5-7 support, because it's missing - // Object.prototype.constructor. Remove this eventually - function isBuffer(obj) { - return obj != null && (!!obj._isBuffer || isFastBuffer(obj) || isSlowBuffer(obj)) - } - - function isFastBuffer (obj) { - return !!obj.constructor && typeof obj.constructor.isBuffer === 'function' && obj.constructor.isBuffer(obj) - } - - // For Node v0.10 support. Remove this eventually. - function isSlowBuffer (obj) { - return typeof obj.readFloatLE === 'function' && typeof obj.slice === 'function' && isFastBuffer(obj.slice(0, 0)) - } - - var domain; - - // This constructor is used to store event handlers. Instantiating this is - // faster than explicitly calling `Object.create(null)` to get a "clean" empty - // object (tested with v8 v4.9). - function EventHandlers() {} - EventHandlers.prototype = Object.create(null); - - function EventEmitter() { - EventEmitter.init.call(this); - } - - // nodejs oddity - // require('events') === require('events').EventEmitter - EventEmitter.EventEmitter = EventEmitter; - - EventEmitter.usingDomains = false; - - EventEmitter.prototype.domain = undefined; - EventEmitter.prototype._events = undefined; - EventEmitter.prototype._maxListeners = undefined; - - // By default EventEmitters will print a warning if more than 10 listeners are - // added to it. This is a useful default which helps finding memory leaks. - EventEmitter.defaultMaxListeners = 10; - - EventEmitter.init = function() { - this.domain = null; - if (EventEmitter.usingDomains) { - // if there is an active domain, then attach to it. - if (domain.active ) ; - } - - if (!this._events || this._events === Object.getPrototypeOf(this)._events) { - this._events = new EventHandlers(); - this._eventsCount = 0; - } - - this._maxListeners = this._maxListeners || undefined; - }; - - // Obviously not all Emitters should be limited to 10. This function allows - // that to be increased. Set to zero for unlimited. - EventEmitter.prototype.setMaxListeners = function setMaxListeners(n) { - if (typeof n !== 'number' || n < 0 || isNaN(n)) - throw new TypeError('"n" argument must be a positive number'); - this._maxListeners = n; - return this; - }; - - function $getMaxListeners(that) { - if (that._maxListeners === undefined) - return EventEmitter.defaultMaxListeners; - return that._maxListeners; - } - - EventEmitter.prototype.getMaxListeners = function getMaxListeners() { - return $getMaxListeners(this); - }; - - // These standalone emit* functions are used to optimize calling of event - // handlers for fast cases because emit() itself often has a variable number of - // arguments and can be deoptimized because of that. These functions always have - // the same number of arguments and thus do not get deoptimized, so the code - // inside them can execute faster. - function emitNone(handler, isFn, self) { - if (isFn) - handler.call(self); - else { - var len = handler.length; - var listeners = arrayClone(handler, len); - for (var i = 0; i < len; ++i) - listeners[i].call(self); - } - } - function emitOne(handler, isFn, self, arg1) { - if (isFn) - handler.call(self, arg1); - else { - var len = handler.length; - var listeners = arrayClone(handler, len); - for (var i = 0; i < len; ++i) - listeners[i].call(self, arg1); - } - } - function emitTwo(handler, isFn, self, arg1, arg2) { - if (isFn) - handler.call(self, arg1, arg2); - else { - var len = handler.length; - var listeners = arrayClone(handler, len); - for (var i = 0; i < len; ++i) - listeners[i].call(self, arg1, arg2); - } - } - function emitThree(handler, isFn, self, arg1, arg2, arg3) { - if (isFn) - handler.call(self, arg1, arg2, arg3); - else { - var len = handler.length; - var listeners = arrayClone(handler, len); - for (var i = 0; i < len; ++i) - listeners[i].call(self, arg1, arg2, arg3); - } - } - - function emitMany(handler, isFn, self, args) { - if (isFn) - handler.apply(self, args); - else { - var len = handler.length; - var listeners = arrayClone(handler, len); - for (var i = 0; i < len; ++i) - listeners[i].apply(self, args); - } - } - - EventEmitter.prototype.emit = function emit(type) { - var er, handler, len, args, i, events, domain; - var doError = (type === 'error'); - - events = this._events; - if (events) - doError = (doError && events.error == null); - else if (!doError) - return false; - - domain = this.domain; - - // If there is no 'error' event listener then throw. - if (doError) { - er = arguments[1]; - if (domain) { - if (!er) - er = new Error('Uncaught, unspecified "error" event'); - er.domainEmitter = this; - er.domain = domain; - er.domainThrown = false; - domain.emit('error', er); - } else if (er instanceof Error) { - throw er; // Unhandled 'error' event - } else { - // At least give some kind of context to the user - var err = new Error('Uncaught, unspecified "error" event. (' + er + ')'); - err.context = er; - throw err; - } - return false; - } - - handler = events[type]; - - if (!handler) - return false; - - var isFn = typeof handler === 'function'; - len = arguments.length; - switch (len) { - // fast cases - case 1: - emitNone(handler, isFn, this); - break; - case 2: - emitOne(handler, isFn, this, arguments[1]); - break; - case 3: - emitTwo(handler, isFn, this, arguments[1], arguments[2]); - break; - case 4: - emitThree(handler, isFn, this, arguments[1], arguments[2], arguments[3]); - break; - // slower - default: - args = new Array(len - 1); - for (i = 1; i < len; i++) - args[i - 1] = arguments[i]; - emitMany(handler, isFn, this, args); - } - - return true; - }; - - function _addListener(target, type, listener, prepend) { - var m; - var events; - var existing; - - if (typeof listener !== 'function') - throw new TypeError('"listener" argument must be a function'); - - events = target._events; - if (!events) { - events = target._events = new EventHandlers(); - target._eventsCount = 0; - } else { - // To avoid recursion in the case that type === "newListener"! Before - // adding it to the listeners, first emit "newListener". - if (events.newListener) { - target.emit('newListener', type, - listener.listener ? listener.listener : listener); - - // Re-assign `events` because a newListener handler could have caused the - // this._events to be assigned to a new object - events = target._events; - } - existing = events[type]; - } - - if (!existing) { - // Optimize the case of one listener. Don't need the extra array object. - existing = events[type] = listener; - ++target._eventsCount; - } else { - if (typeof existing === 'function') { - // Adding the second element, need to change to array. - existing = events[type] = prepend ? [listener, existing] : - [existing, listener]; - } else { - // If we've already got an array, just append. - if (prepend) { - existing.unshift(listener); - } else { - existing.push(listener); - } - } - - // Check for listener leak - if (!existing.warned) { - m = $getMaxListeners(target); - if (m && m > 0 && existing.length > m) { - existing.warned = true; - var w = new Error('Possible EventEmitter memory leak detected. ' + - existing.length + ' ' + type + ' listeners added. ' + - 'Use emitter.setMaxListeners() to increase limit'); - w.name = 'MaxListenersExceededWarning'; - w.emitter = target; - w.type = type; - w.count = existing.length; - emitWarning(w); - } - } - } - - return target; - } - function emitWarning(e) { - typeof console.warn === 'function' ? console.warn(e) : console.log(e); - } - EventEmitter.prototype.addListener = function addListener(type, listener) { - return _addListener(this, type, listener, false); - }; - - EventEmitter.prototype.on = EventEmitter.prototype.addListener; - - EventEmitter.prototype.prependListener = - function prependListener(type, listener) { - return _addListener(this, type, listener, true); - }; - - function _onceWrap(target, type, listener) { - var fired = false; - function g() { - target.removeListener(type, g); - if (!fired) { - fired = true; - listener.apply(target, arguments); - } - } - g.listener = listener; - return g; - } - - EventEmitter.prototype.once = function once(type, listener) { - if (typeof listener !== 'function') - throw new TypeError('"listener" argument must be a function'); - this.on(type, _onceWrap(this, type, listener)); - return this; - }; - - EventEmitter.prototype.prependOnceListener = - function prependOnceListener(type, listener) { - if (typeof listener !== 'function') - throw new TypeError('"listener" argument must be a function'); - this.prependListener(type, _onceWrap(this, type, listener)); - return this; - }; - - // emits a 'removeListener' event iff the listener was removed - EventEmitter.prototype.removeListener = - function removeListener(type, listener) { - var list, events, position, i, originalListener; - - if (typeof listener !== 'function') - throw new TypeError('"listener" argument must be a function'); - - events = this._events; - if (!events) - return this; - - list = events[type]; - if (!list) - return this; - - if (list === listener || (list.listener && list.listener === listener)) { - if (--this._eventsCount === 0) - this._events = new EventHandlers(); - else { - delete events[type]; - if (events.removeListener) - this.emit('removeListener', type, list.listener || listener); - } - } else if (typeof list !== 'function') { - position = -1; - - for (i = list.length; i-- > 0;) { - if (list[i] === listener || - (list[i].listener && list[i].listener === listener)) { - originalListener = list[i].listener; - position = i; - break; - } - } - - if (position < 0) - return this; - - if (list.length === 1) { - list[0] = undefined; - if (--this._eventsCount === 0) { - this._events = new EventHandlers(); - return this; - } else { - delete events[type]; - } - } else { - spliceOne(list, position); - } - - if (events.removeListener) - this.emit('removeListener', type, originalListener || listener); - } - - return this; - }; - - EventEmitter.prototype.removeAllListeners = - function removeAllListeners(type) { - var listeners, events; - - events = this._events; - if (!events) - return this; - - // not listening for removeListener, no need to emit - if (!events.removeListener) { - if (arguments.length === 0) { - this._events = new EventHandlers(); - this._eventsCount = 0; - } else if (events[type]) { - if (--this._eventsCount === 0) - this._events = new EventHandlers(); - else - delete events[type]; - } - return this; - } - - // emit removeListener for all listeners on all events - if (arguments.length === 0) { - var keys = Object.keys(events); - for (var i = 0, key; i < keys.length; ++i) { - key = keys[i]; - if (key === 'removeListener') continue; - this.removeAllListeners(key); - } - this.removeAllListeners('removeListener'); - this._events = new EventHandlers(); - this._eventsCount = 0; - return this; - } - - listeners = events[type]; - - if (typeof listeners === 'function') { - this.removeListener(type, listeners); - } else if (listeners) { - // LIFO order - do { - this.removeListener(type, listeners[listeners.length - 1]); - } while (listeners[0]); - } - - return this; - }; - - EventEmitter.prototype.listeners = function listeners(type) { - var evlistener; - var ret; - var events = this._events; - - if (!events) - ret = []; - else { - evlistener = events[type]; - if (!evlistener) - ret = []; - else if (typeof evlistener === 'function') - ret = [evlistener.listener || evlistener]; - else - ret = unwrapListeners(evlistener); - } - - return ret; - }; - - EventEmitter.listenerCount = function(emitter, type) { - if (typeof emitter.listenerCount === 'function') { - return emitter.listenerCount(type); - } else { - return listenerCount$1.call(emitter, type); - } - }; - - EventEmitter.prototype.listenerCount = listenerCount$1; - function listenerCount$1(type) { - var events = this._events; - - if (events) { - var evlistener = events[type]; - - if (typeof evlistener === 'function') { - return 1; - } else if (evlistener) { - return evlistener.length; - } - } - - return 0; - } - - EventEmitter.prototype.eventNames = function eventNames() { - return this._eventsCount > 0 ? Reflect.ownKeys(this._events) : []; - }; - - // About 1.5x faster than the two-arg version of Array#splice(). - function spliceOne(list, index) { - for (var i = index, k = i + 1, n = list.length; k < n; i += 1, k += 1) - list[i] = list[k]; - list.pop(); - } - - function arrayClone(arr, i) { - var copy = new Array(i); - while (i--) - copy[i] = arr[i]; - return copy; - } - - function unwrapListeners(arr) { - var ret = new Array(arr.length); - for (var i = 0; i < ret.length; ++i) { - ret[i] = arr[i].listener || arr[i]; - } - return ret; - } - - // shim for using process in browser - // based off https://github.com/defunctzombie/node-process/blob/master/browser.js - - function defaultSetTimout() { - throw new Error('setTimeout has not been defined'); - } - function defaultClearTimeout () { - throw new Error('clearTimeout has not been defined'); - } - var cachedSetTimeout = defaultSetTimout; - var cachedClearTimeout = defaultClearTimeout; - if (typeof global$1.setTimeout === 'function') { - cachedSetTimeout = setTimeout; - } - if (typeof global$1.clearTimeout === 'function') { - cachedClearTimeout = clearTimeout; - } - - function runTimeout(fun) { - if (cachedSetTimeout === setTimeout) { - //normal enviroments in sane situations - return setTimeout(fun, 0); - } - // if setTimeout wasn't available but was latter defined - if ((cachedSetTimeout === defaultSetTimout || !cachedSetTimeout) && setTimeout) { - cachedSetTimeout = setTimeout; - return setTimeout(fun, 0); - } - try { - // when when somebody has screwed with setTimeout but no I.E. maddness - return cachedSetTimeout(fun, 0); - } catch(e){ - try { - // When we are in I.E. but the script has been evaled so I.E. doesn't trust the global object when called normally - return cachedSetTimeout.call(null, fun, 0); - } catch(e){ - // same as above but when it's a version of I.E. that must have the global object for 'this', hopfully our context correct otherwise it will throw a global error - return cachedSetTimeout.call(this, fun, 0); - } - } - - - } - function runClearTimeout(marker) { - if (cachedClearTimeout === clearTimeout) { - //normal enviroments in sane situations - return clearTimeout(marker); - } - // if clearTimeout wasn't available but was latter defined - if ((cachedClearTimeout === defaultClearTimeout || !cachedClearTimeout) && clearTimeout) { - cachedClearTimeout = clearTimeout; - return clearTimeout(marker); - } - try { - // when when somebody has screwed with setTimeout but no I.E. maddness - return cachedClearTimeout(marker); - } catch (e){ - try { - // When we are in I.E. but the script has been evaled so I.E. doesn't trust the global object when called normally - return cachedClearTimeout.call(null, marker); - } catch (e){ - // same as above but when it's a version of I.E. that must have the global object for 'this', hopfully our context correct otherwise it will throw a global error. - // Some versions of I.E. have different rules for clearTimeout vs setTimeout - return cachedClearTimeout.call(this, marker); - } - } - - - - } - var queue = []; - var draining = false; - var currentQueue; - var queueIndex = -1; - - function cleanUpNextTick() { - if (!draining || !currentQueue) { - return; - } - draining = false; - if (currentQueue.length) { - queue = currentQueue.concat(queue); - } else { - queueIndex = -1; - } - if (queue.length) { - drainQueue(); - } - } - - function drainQueue() { - if (draining) { - return; - } - var timeout = runTimeout(cleanUpNextTick); - draining = true; - - var len = queue.length; - while(len) { - currentQueue = queue; - queue = []; - while (++queueIndex < len) { - if (currentQueue) { - currentQueue[queueIndex].run(); - } - } - queueIndex = -1; - len = queue.length; - } - currentQueue = null; - draining = false; - runClearTimeout(timeout); - } - function nextTick(fun) { - var args = new Array(arguments.length - 1); - if (arguments.length > 1) { - for (var i = 1; i < arguments.length; i++) { - args[i - 1] = arguments[i]; - } - } - queue.push(new Item(fun, args)); - if (queue.length === 1 && !draining) { - runTimeout(drainQueue); - } - } - // v8 likes predictible objects - function Item(fun, array) { - this.fun = fun; - this.array = array; - } - Item.prototype.run = function () { - this.fun.apply(null, this.array); - }; - - // from https://github.com/kumavis/browser-process-hrtime/blob/master/index.js - var performance = global$1.performance || {}; - performance.now || - performance.mozNow || - performance.msNow || - performance.oNow || - performance.webkitNow || - function(){ return (new Date()).getTime() }; - - var inherits; - if (typeof Object.create === 'function'){ - inherits = function inherits(ctor, superCtor) { - // implementation from standard node.js 'util' module - ctor.super_ = superCtor; - ctor.prototype = Object.create(superCtor.prototype, { - constructor: { - value: ctor, - enumerable: false, - writable: true, - configurable: true - } - }); - }; - } else { - inherits = function inherits(ctor, superCtor) { - ctor.super_ = superCtor; - var TempCtor = function () {}; - TempCtor.prototype = superCtor.prototype; - ctor.prototype = new TempCtor(); - ctor.prototype.constructor = ctor; - }; - } - var inherits$1 = inherits; - - var formatRegExp = /%[sdj%]/g; - function format(f) { - if (!isString(f)) { - var objects = []; - for (var i = 0; i < arguments.length; i++) { - objects.push(inspect(arguments[i])); - } - return objects.join(' '); - } - - var i = 1; - var args = arguments; - var len = args.length; - var str = String(f).replace(formatRegExp, function(x) { - if (x === '%%') return '%'; - if (i >= len) return x; - switch (x) { - case '%s': return String(args[i++]); - case '%d': return Number(args[i++]); - case '%j': - try { - return JSON.stringify(args[i++]); - } catch (_) { - return '[Circular]'; - } - default: - return x; - } - }); - for (var x = args[i]; i < len; x = args[++i]) { - if (isNull(x) || !isObject$1(x)) { - str += ' ' + x; - } else { - str += ' ' + inspect(x); - } - } - return str; - } - - // Mark that a method should not be used. - // Returns a modified function which warns once by default. - // If --no-deprecation is set, then it is a no-op. - function deprecate(fn, msg) { - // Allow for deprecating things in the process of starting up. - if (isUndefined(global$1.process)) { - return function() { - return deprecate(fn, msg).apply(this, arguments); - }; - } - - var warned = false; - function deprecated() { - if (!warned) { - { - console.error(msg); - } - warned = true; - } - return fn.apply(this, arguments); - } - - return deprecated; - } - - var debugs = {}; - var debugEnviron; - function debuglog(set) { - if (isUndefined(debugEnviron)) - debugEnviron = ''; - set = set.toUpperCase(); - if (!debugs[set]) { - if (new RegExp('\\b' + set + '\\b', 'i').test(debugEnviron)) { - var pid = 0; - debugs[set] = function() { - var msg = format.apply(null, arguments); - console.error('%s %d: %s', set, pid, msg); - }; - } else { - debugs[set] = function() {}; - } - } - return debugs[set]; - } - - /** - * Echos the value of a value. Trys to print the value out - * in the best way possible given the different types. - * - * @param {Object} obj The object to print out. - * @param {Object} opts Optional options object that alters the output. - */ - /* legacy: obj, showHidden, depth, colors*/ - function inspect(obj, opts) { - // default options - var ctx = { - seen: [], - stylize: stylizeNoColor - }; - // legacy... - if (arguments.length >= 3) ctx.depth = arguments[2]; - if (arguments.length >= 4) ctx.colors = arguments[3]; - if (isBoolean(opts)) { - // legacy... - ctx.showHidden = opts; - } else if (opts) { - // got an "options" object - _extend(ctx, opts); - } - // set default options - if (isUndefined(ctx.showHidden)) ctx.showHidden = false; - if (isUndefined(ctx.depth)) ctx.depth = 2; - if (isUndefined(ctx.colors)) ctx.colors = false; - if (isUndefined(ctx.customInspect)) ctx.customInspect = true; - if (ctx.colors) ctx.stylize = stylizeWithColor; - return formatValue(ctx, obj, ctx.depth); - } - - // http://en.wikipedia.org/wiki/ANSI_escape_code#graphics - inspect.colors = { - 'bold' : [1, 22], - 'italic' : [3, 23], - 'underline' : [4, 24], - 'inverse' : [7, 27], - 'white' : [37, 39], - 'grey' : [90, 39], - 'black' : [30, 39], - 'blue' : [34, 39], - 'cyan' : [36, 39], - 'green' : [32, 39], - 'magenta' : [35, 39], - 'red' : [31, 39], - 'yellow' : [33, 39] - }; - - // Don't use 'blue' not visible on cmd.exe - inspect.styles = { - 'special': 'cyan', - 'number': 'yellow', - 'boolean': 'yellow', - 'undefined': 'grey', - 'null': 'bold', - 'string': 'green', - 'date': 'magenta', - // "name": intentionally not styling - 'regexp': 'red' - }; - - - function stylizeWithColor(str, styleType) { - var style = inspect.styles[styleType]; - - if (style) { - return '\u001b[' + inspect.colors[style][0] + 'm' + str + - '\u001b[' + inspect.colors[style][1] + 'm'; - } else { - return str; - } - } - - - function stylizeNoColor(str, styleType) { - return str; - } - - - function arrayToHash(array) { - var hash = {}; - - array.forEach(function(val, idx) { - hash[val] = true; - }); - - return hash; - } - - - function formatValue(ctx, value, recurseTimes) { - // Provide a hook for user-specified inspect functions. - // Check that value is an object with an inspect function on it - if (ctx.customInspect && - value && - isFunction(value.inspect) && - // Filter out the util module, it's inspect function is special - value.inspect !== inspect && - // Also filter out any prototype objects using the circular check. - !(value.constructor && value.constructor.prototype === value)) { - var ret = value.inspect(recurseTimes, ctx); - if (!isString(ret)) { - ret = formatValue(ctx, ret, recurseTimes); - } - return ret; - } - - // Primitive types cannot have properties - var primitive = formatPrimitive(ctx, value); - if (primitive) { - return primitive; - } - - // Look up the keys of the object. - var keys = Object.keys(value); - var visibleKeys = arrayToHash(keys); - - if (ctx.showHidden) { - keys = Object.getOwnPropertyNames(value); - } - - // IE doesn't make error fields non-enumerable - // http://msdn.microsoft.com/en-us/library/ie/dww52sbt(v=vs.94).aspx - if (isError(value) - && (keys.indexOf('message') >= 0 || keys.indexOf('description') >= 0)) { - return formatError(value); - } - - // Some type of object without properties can be shortcutted. - if (keys.length === 0) { - if (isFunction(value)) { - var name = value.name ? ': ' + value.name : ''; - return ctx.stylize('[Function' + name + ']', 'special'); - } - if (isRegExp(value)) { - return ctx.stylize(RegExp.prototype.toString.call(value), 'regexp'); - } - if (isDate(value)) { - return ctx.stylize(Date.prototype.toString.call(value), 'date'); - } - if (isError(value)) { - return formatError(value); - } - } - - var base = '', array = false, braces = ['{', '}']; - - // Make Array say that they are Array - if (isArray(value)) { - array = true; - braces = ['[', ']']; - } - - // Make functions say that they are functions - if (isFunction(value)) { - var n = value.name ? ': ' + value.name : ''; - base = ' [Function' + n + ']'; - } - - // Make RegExps say that they are RegExps - if (isRegExp(value)) { - base = ' ' + RegExp.prototype.toString.call(value); - } - - // Make dates with properties first say the date - if (isDate(value)) { - base = ' ' + Date.prototype.toUTCString.call(value); - } - - // Make error with message first say the error - if (isError(value)) { - base = ' ' + formatError(value); - } - - if (keys.length === 0 && (!array || value.length == 0)) { - return braces[0] + base + braces[1]; - } - - if (recurseTimes < 0) { - if (isRegExp(value)) { - return ctx.stylize(RegExp.prototype.toString.call(value), 'regexp'); - } else { - return ctx.stylize('[Object]', 'special'); - } - } - - ctx.seen.push(value); - - var output; - if (array) { - output = formatArray(ctx, value, recurseTimes, visibleKeys, keys); - } else { - output = keys.map(function(key) { - return formatProperty(ctx, value, recurseTimes, visibleKeys, key, array); - }); - } - - ctx.seen.pop(); - - return reduceToSingleString(output, base, braces); - } - - - function formatPrimitive(ctx, value) { - if (isUndefined(value)) - return ctx.stylize('undefined', 'undefined'); - if (isString(value)) { - var simple = '\'' + JSON.stringify(value).replace(/^"|"$/g, '') - .replace(/'/g, "\\'") - .replace(/\\"/g, '"') + '\''; - return ctx.stylize(simple, 'string'); - } - if (isNumber(value)) - return ctx.stylize('' + value, 'number'); - if (isBoolean(value)) - return ctx.stylize('' + value, 'boolean'); - // For some reason typeof null is "object", so special case here. - if (isNull(value)) - return ctx.stylize('null', 'null'); - } - - - function formatError(value) { - return '[' + Error.prototype.toString.call(value) + ']'; - } - - - function formatArray(ctx, value, recurseTimes, visibleKeys, keys) { - var output = []; - for (var i = 0, l = value.length; i < l; ++i) { - if (hasOwnProperty(value, String(i))) { - output.push(formatProperty(ctx, value, recurseTimes, visibleKeys, - String(i), true)); - } else { - output.push(''); - } - } - keys.forEach(function(key) { - if (!key.match(/^\d+$/)) { - output.push(formatProperty(ctx, value, recurseTimes, visibleKeys, - key, true)); - } - }); - return output; - } - - - function formatProperty(ctx, value, recurseTimes, visibleKeys, key, array) { - var name, str, desc; - desc = Object.getOwnPropertyDescriptor(value, key) || { value: value[key] }; - if (desc.get) { - if (desc.set) { - str = ctx.stylize('[Getter/Setter]', 'special'); - } else { - str = ctx.stylize('[Getter]', 'special'); - } - } else { - if (desc.set) { - str = ctx.stylize('[Setter]', 'special'); - } - } - if (!hasOwnProperty(visibleKeys, key)) { - name = '[' + key + ']'; - } - if (!str) { - if (ctx.seen.indexOf(desc.value) < 0) { - if (isNull(recurseTimes)) { - str = formatValue(ctx, desc.value, null); - } else { - str = formatValue(ctx, desc.value, recurseTimes - 1); - } - if (str.indexOf('\n') > -1) { - if (array) { - str = str.split('\n').map(function(line) { - return ' ' + line; - }).join('\n').substr(2); - } else { - str = '\n' + str.split('\n').map(function(line) { - return ' ' + line; - }).join('\n'); - } - } - } else { - str = ctx.stylize('[Circular]', 'special'); - } - } - if (isUndefined(name)) { - if (array && key.match(/^\d+$/)) { - return str; - } - name = JSON.stringify('' + key); - if (name.match(/^"([a-zA-Z_][a-zA-Z_0-9]*)"$/)) { - name = name.substr(1, name.length - 2); - name = ctx.stylize(name, 'name'); - } else { - name = name.replace(/'/g, "\\'") - .replace(/\\"/g, '"') - .replace(/(^"|"$)/g, "'"); - name = ctx.stylize(name, 'string'); - } - } - - return name + ': ' + str; - } - - - function reduceToSingleString(output, base, braces) { - var length = output.reduce(function(prev, cur) { - if (cur.indexOf('\n') >= 0) ; - return prev + cur.replace(/\u001b\[\d\d?m/g, '').length + 1; - }, 0); - - if (length > 60) { - return braces[0] + - (base === '' ? '' : base + '\n ') + - ' ' + - output.join(',\n ') + - ' ' + - braces[1]; - } - - return braces[0] + base + ' ' + output.join(', ') + ' ' + braces[1]; - } - - - // NOTE: These type checking functions intentionally don't use `instanceof` - // because it is fragile and can be easily faked with `Object.create()`. - function isArray(ar) { - return Array.isArray(ar); - } - - function isBoolean(arg) { - return typeof arg === 'boolean'; - } - - function isNull(arg) { - return arg === null; - } - - function isNumber(arg) { - return typeof arg === 'number'; - } - - function isString(arg) { - return typeof arg === 'string'; - } - - function isUndefined(arg) { - return arg === void 0; - } - - function isRegExp(re) { - return isObject$1(re) && objectToString(re) === '[object RegExp]'; - } - - function isObject$1(arg) { - return typeof arg === 'object' && arg !== null; - } - - function isDate(d) { - return isObject$1(d) && objectToString(d) === '[object Date]'; - } - - function isError(e) { - return isObject$1(e) && - (objectToString(e) === '[object Error]' || e instanceof Error); - } - - function isFunction(arg) { - return typeof arg === 'function'; - } - - function objectToString(o) { - return Object.prototype.toString.call(o); - } - - function _extend(origin, add) { - // Don't do anything if add isn't an object - if (!add || !isObject$1(add)) return origin; - - var keys = Object.keys(add); - var i = keys.length; - while (i--) { - origin[keys[i]] = add[keys[i]]; - } - return origin; - } - function hasOwnProperty(obj, prop) { - return Object.prototype.hasOwnProperty.call(obj, prop); - } - - function BufferList() { - this.head = null; - this.tail = null; - this.length = 0; - } - - BufferList.prototype.push = function (v) { - var entry = { data: v, next: null }; - if (this.length > 0) this.tail.next = entry;else this.head = entry; - this.tail = entry; - ++this.length; - }; - - BufferList.prototype.unshift = function (v) { - var entry = { data: v, next: this.head }; - if (this.length === 0) this.tail = entry; - this.head = entry; - ++this.length; - }; - - BufferList.prototype.shift = function () { - if (this.length === 0) return; - var ret = this.head.data; - if (this.length === 1) this.head = this.tail = null;else this.head = this.head.next; - --this.length; - return ret; - }; - - BufferList.prototype.clear = function () { - this.head = this.tail = null; - this.length = 0; - }; - - BufferList.prototype.join = function (s) { - if (this.length === 0) return ''; - var p = this.head; - var ret = '' + p.data; - while (p = p.next) { - ret += s + p.data; - }return ret; - }; - - BufferList.prototype.concat = function (n) { - if (this.length === 0) return Buffer.alloc(0); - if (this.length === 1) return this.head.data; - var ret = Buffer.allocUnsafe(n >>> 0); - var p = this.head; - var i = 0; - while (p) { - p.data.copy(ret, i); - i += p.data.length; - p = p.next; - } - return ret; - }; - - // Copyright Joyent, Inc. and other Node contributors. - var isBufferEncoding = Buffer.isEncoding - || function(encoding) { - switch (encoding && encoding.toLowerCase()) { - case 'hex': case 'utf8': case 'utf-8': case 'ascii': case 'binary': case 'base64': case 'ucs2': case 'ucs-2': case 'utf16le': case 'utf-16le': case 'raw': return true; - default: return false; - } - }; - - - function assertEncoding(encoding) { - if (encoding && !isBufferEncoding(encoding)) { - throw new Error('Unknown encoding: ' + encoding); - } - } - - // StringDecoder provides an interface for efficiently splitting a series of - // buffers into a series of JS strings without breaking apart multi-byte - // characters. CESU-8 is handled as part of the UTF-8 encoding. - // - // @TODO Handling all encodings inside a single object makes it very difficult - // to reason about this code, so it should be split up in the future. - // @TODO There should be a utf8-strict encoding that rejects invalid UTF-8 code - // points as used by CESU-8. - function StringDecoder(encoding) { - this.encoding = (encoding || 'utf8').toLowerCase().replace(/[-_]/, ''); - assertEncoding(encoding); - switch (this.encoding) { - case 'utf8': - // CESU-8 represents each of Surrogate Pair by 3-bytes - this.surrogateSize = 3; - break; - case 'ucs2': - case 'utf16le': - // UTF-16 represents each of Surrogate Pair by 2-bytes - this.surrogateSize = 2; - this.detectIncompleteChar = utf16DetectIncompleteChar; - break; - case 'base64': - // Base-64 stores 3 bytes in 4 chars, and pads the remainder. - this.surrogateSize = 3; - this.detectIncompleteChar = base64DetectIncompleteChar; - break; - default: - this.write = passThroughWrite; - return; - } - - // Enough space to store all bytes of a single character. UTF-8 needs 4 - // bytes, but CESU-8 may require up to 6 (3 bytes per surrogate). - this.charBuffer = new Buffer(6); - // Number of bytes received for the current incomplete multi-byte character. - this.charReceived = 0; - // Number of bytes expected for the current incomplete multi-byte character. - this.charLength = 0; - } - - // write decodes the given buffer and returns it as JS string that is - // guaranteed to not contain any partial multi-byte characters. Any partial - // character found at the end of the buffer is buffered up, and will be - // returned when calling write again with the remaining bytes. - // - // Note: Converting a Buffer containing an orphan surrogate to a String - // currently works, but converting a String to a Buffer (via `new Buffer`, or - // Buffer#write) will replace incomplete surrogates with the unicode - // replacement character. See https://codereview.chromium.org/121173009/ . - StringDecoder.prototype.write = function(buffer) { - var charStr = ''; - // if our last write ended with an incomplete multibyte character - while (this.charLength) { - // determine how many remaining bytes this buffer has to offer for this char - var available = (buffer.length >= this.charLength - this.charReceived) ? - this.charLength - this.charReceived : - buffer.length; - - // add the new bytes to the char buffer - buffer.copy(this.charBuffer, this.charReceived, 0, available); - this.charReceived += available; - - if (this.charReceived < this.charLength) { - // still not enough chars in this buffer? wait for more ... - return ''; - } - - // remove bytes belonging to the current character from the buffer - buffer = buffer.slice(available, buffer.length); - - // get the character that was split - charStr = this.charBuffer.slice(0, this.charLength).toString(this.encoding); - - // CESU-8: lead surrogate (D800-DBFF) is also the incomplete character - var charCode = charStr.charCodeAt(charStr.length - 1); - if (charCode >= 0xD800 && charCode <= 0xDBFF) { - this.charLength += this.surrogateSize; - charStr = ''; - continue; - } - this.charReceived = this.charLength = 0; - - // if there are no more bytes in this buffer, just emit our char - if (buffer.length === 0) { - return charStr; - } - break; - } - - // determine and set charLength / charReceived - this.detectIncompleteChar(buffer); - - var end = buffer.length; - if (this.charLength) { - // buffer the incomplete character bytes we got - buffer.copy(this.charBuffer, 0, buffer.length - this.charReceived, end); - end -= this.charReceived; - } - - charStr += buffer.toString(this.encoding, 0, end); - - var end = charStr.length - 1; - var charCode = charStr.charCodeAt(end); - // CESU-8: lead surrogate (D800-DBFF) is also the incomplete character - if (charCode >= 0xD800 && charCode <= 0xDBFF) { - var size = this.surrogateSize; - this.charLength += size; - this.charReceived += size; - this.charBuffer.copy(this.charBuffer, size, 0, size); - buffer.copy(this.charBuffer, 0, 0, size); - return charStr.substring(0, end); - } - - // or just emit the charStr - return charStr; - }; - - // detectIncompleteChar determines if there is an incomplete UTF-8 character at - // the end of the given buffer. If so, it sets this.charLength to the byte - // length that character, and sets this.charReceived to the number of bytes - // that are available for this character. - StringDecoder.prototype.detectIncompleteChar = function(buffer) { - // determine how many bytes we have to check at the end of this buffer - var i = (buffer.length >= 3) ? 3 : buffer.length; - - // Figure out if one of the last i bytes of our buffer announces an - // incomplete char. - for (; i > 0; i--) { - var c = buffer[buffer.length - i]; - - // See http://en.wikipedia.org/wiki/UTF-8#Description - - // 110XXXXX - if (i == 1 && c >> 5 == 0x06) { - this.charLength = 2; - break; - } - - // 1110XXXX - if (i <= 2 && c >> 4 == 0x0E) { - this.charLength = 3; - break; - } - - // 11110XXX - if (i <= 3 && c >> 3 == 0x1E) { - this.charLength = 4; - break; - } - } - this.charReceived = i; - }; - - StringDecoder.prototype.end = function(buffer) { - var res = ''; - if (buffer && buffer.length) - res = this.write(buffer); - - if (this.charReceived) { - var cr = this.charReceived; - var buf = this.charBuffer; - var enc = this.encoding; - res += buf.slice(0, cr).toString(enc); - } - - return res; - }; - - function passThroughWrite(buffer) { - return buffer.toString(this.encoding); - } - - function utf16DetectIncompleteChar(buffer) { - this.charReceived = buffer.length % 2; - this.charLength = this.charReceived ? 2 : 0; - } - - function base64DetectIncompleteChar(buffer) { - this.charReceived = buffer.length % 3; - this.charLength = this.charReceived ? 3 : 0; - } - - Readable.ReadableState = ReadableState; - - var debug = debuglog('stream'); - inherits$1(Readable, EventEmitter); - - function prependListener(emitter, event, fn) { - // Sadly this is not cacheable as some libraries bundle their own - // event emitter implementation with them. - if (typeof emitter.prependListener === 'function') { - return emitter.prependListener(event, fn); - } else { - // This is a hack to make sure that our error handler is attached before any - // userland ones. NEVER DO THIS. This is here only because this code needs - // to continue to work with older versions of Node.js that do not include - // the prependListener() method. The goal is to eventually remove this hack. - if (!emitter._events || !emitter._events[event]) - emitter.on(event, fn); - else if (Array.isArray(emitter._events[event])) - emitter._events[event].unshift(fn); - else - emitter._events[event] = [fn, emitter._events[event]]; - } - } - function listenerCount (emitter, type) { - return emitter.listeners(type).length; - } - function ReadableState(options, stream) { - - options = options || {}; - - // object stream flag. Used to make read(n) ignore n and to - // make all the buffer merging and length checks go away - this.objectMode = !!options.objectMode; - - if (stream instanceof Duplex) this.objectMode = this.objectMode || !!options.readableObjectMode; - - // the point at which it stops calling _read() to fill the buffer - // Note: 0 is a valid value, means "don't call _read preemptively ever" - var hwm = options.highWaterMark; - var defaultHwm = this.objectMode ? 16 : 16 * 1024; - this.highWaterMark = hwm || hwm === 0 ? hwm : defaultHwm; - - // cast to ints. - this.highWaterMark = ~ ~this.highWaterMark; - - // A linked list is used to store data chunks instead of an array because the - // linked list can remove elements from the beginning faster than - // array.shift() - this.buffer = new BufferList(); - this.length = 0; - this.pipes = null; - this.pipesCount = 0; - this.flowing = null; - this.ended = false; - this.endEmitted = false; - this.reading = false; - - // a flag to be able to tell if the onwrite cb is called immediately, - // or on a later tick. We set this to true at first, because any - // actions that shouldn't happen until "later" should generally also - // not happen before the first write call. - this.sync = true; - - // whenever we return null, then we set a flag to say - // that we're awaiting a 'readable' event emission. - this.needReadable = false; - this.emittedReadable = false; - this.readableListening = false; - this.resumeScheduled = false; - - // Crypto is kind of old and crusty. Historically, its default string - // encoding is 'binary' so we have to make this configurable. - // Everything else in the universe uses 'utf8', though. - this.defaultEncoding = options.defaultEncoding || 'utf8'; - - // when piping, we only care about 'readable' events that happen - // after read()ing all the bytes and not getting any pushback. - this.ranOut = false; - - // the number of writers that are awaiting a drain event in .pipe()s - this.awaitDrain = 0; - - // if true, a maybeReadMore has been scheduled - this.readingMore = false; - - this.decoder = null; - this.encoding = null; - if (options.encoding) { - this.decoder = new StringDecoder(options.encoding); - this.encoding = options.encoding; - } - } - function Readable(options) { - - if (!(this instanceof Readable)) return new Readable(options); - - this._readableState = new ReadableState(options, this); - - // legacy - this.readable = true; - - if (options && typeof options.read === 'function') this._read = options.read; - - EventEmitter.call(this); - } - - // Manually shove something into the read() buffer. - // This returns true if the highWaterMark has not been hit yet, - // similar to how Writable.write() returns true if you should - // write() some more. - Readable.prototype.push = function (chunk, encoding) { - var state = this._readableState; - - if (!state.objectMode && typeof chunk === 'string') { - encoding = encoding || state.defaultEncoding; - if (encoding !== state.encoding) { - chunk = Buffer.from(chunk, encoding); - encoding = ''; - } - } - - return readableAddChunk(this, state, chunk, encoding, false); - }; - - // Unshift should *always* be something directly out of read() - Readable.prototype.unshift = function (chunk) { - var state = this._readableState; - return readableAddChunk(this, state, chunk, '', true); - }; - - Readable.prototype.isPaused = function () { - return this._readableState.flowing === false; - }; - - function readableAddChunk(stream, state, chunk, encoding, addToFront) { - var er = chunkInvalid(state, chunk); - if (er) { - stream.emit('error', er); - } else if (chunk === null) { - state.reading = false; - onEofChunk(stream, state); - } else if (state.objectMode || chunk && chunk.length > 0) { - if (state.ended && !addToFront) { - var e = new Error('stream.push() after EOF'); - stream.emit('error', e); - } else if (state.endEmitted && addToFront) { - var _e = new Error('stream.unshift() after end event'); - stream.emit('error', _e); - } else { - var skipAdd; - if (state.decoder && !addToFront && !encoding) { - chunk = state.decoder.write(chunk); - skipAdd = !state.objectMode && chunk.length === 0; - } - - if (!addToFront) state.reading = false; - - // Don't add to the buffer if we've decoded to an empty string chunk and - // we're not in object mode - if (!skipAdd) { - // if we want the data now, just emit it. - if (state.flowing && state.length === 0 && !state.sync) { - stream.emit('data', chunk); - stream.read(0); - } else { - // update the buffer info. - state.length += state.objectMode ? 1 : chunk.length; - if (addToFront) state.buffer.unshift(chunk);else state.buffer.push(chunk); - - if (state.needReadable) emitReadable(stream); - } - } - - maybeReadMore(stream, state); - } - } else if (!addToFront) { - state.reading = false; - } - - return needMoreData(state); - } - - // if it's past the high water mark, we can push in some more. - // Also, if we have no data yet, we can stand some - // more bytes. This is to work around cases where hwm=0, - // such as the repl. Also, if the push() triggered a - // readable event, and the user called read(largeNumber) such that - // needReadable was set, then we ought to push more, so that another - // 'readable' event will be triggered. - function needMoreData(state) { - return !state.ended && (state.needReadable || state.length < state.highWaterMark || state.length === 0); - } - - // backwards compatibility. - Readable.prototype.setEncoding = function (enc) { - this._readableState.decoder = new StringDecoder(enc); - this._readableState.encoding = enc; - return this; - }; - - // Don't raise the hwm > 8MB - var MAX_HWM = 0x800000; - function computeNewHighWaterMark(n) { - if (n >= MAX_HWM) { - n = MAX_HWM; - } else { - // Get the next highest power of 2 to prevent increasing hwm excessively in - // tiny amounts - n--; - n |= n >>> 1; - n |= n >>> 2; - n |= n >>> 4; - n |= n >>> 8; - n |= n >>> 16; - n++; - } - return n; - } - - // This function is designed to be inlinable, so please take care when making - // changes to the function body. - function howMuchToRead(n, state) { - if (n <= 0 || state.length === 0 && state.ended) return 0; - if (state.objectMode) return 1; - if (n !== n) { - // Only flow one buffer at a time - if (state.flowing && state.length) return state.buffer.head.data.length;else return state.length; - } - // If we're asking for more than the current hwm, then raise the hwm. - if (n > state.highWaterMark) state.highWaterMark = computeNewHighWaterMark(n); - if (n <= state.length) return n; - // Don't have enough - if (!state.ended) { - state.needReadable = true; - return 0; - } - return state.length; - } - - // you can override either this method, or the async _read(n) below. - Readable.prototype.read = function (n) { - debug('read', n); - n = parseInt(n, 10); - var state = this._readableState; - var nOrig = n; - - if (n !== 0) state.emittedReadable = false; - - // if we're doing read(0) to trigger a readable event, but we - // already have a bunch of data in the buffer, then just trigger - // the 'readable' event and move on. - if (n === 0 && state.needReadable && (state.length >= state.highWaterMark || state.ended)) { - debug('read: emitReadable', state.length, state.ended); - if (state.length === 0 && state.ended) endReadable(this);else emitReadable(this); - return null; - } - - n = howMuchToRead(n, state); - - // if we've ended, and we're now clear, then finish it up. - if (n === 0 && state.ended) { - if (state.length === 0) endReadable(this); - return null; - } - - // All the actual chunk generation logic needs to be - // *below* the call to _read. The reason is that in certain - // synthetic stream cases, such as passthrough streams, _read - // may be a completely synchronous operation which may change - // the state of the read buffer, providing enough data when - // before there was *not* enough. - // - // So, the steps are: - // 1. Figure out what the state of things will be after we do - // a read from the buffer. - // - // 2. If that resulting state will trigger a _read, then call _read. - // Note that this may be asynchronous, or synchronous. Yes, it is - // deeply ugly to write APIs this way, but that still doesn't mean - // that the Readable class should behave improperly, as streams are - // designed to be sync/async agnostic. - // Take note if the _read call is sync or async (ie, if the read call - // has returned yet), so that we know whether or not it's safe to emit - // 'readable' etc. - // - // 3. Actually pull the requested chunks out of the buffer and return. - - // if we need a readable event, then we need to do some reading. - var doRead = state.needReadable; - debug('need readable', doRead); - - // if we currently have less than the highWaterMark, then also read some - if (state.length === 0 || state.length - n < state.highWaterMark) { - doRead = true; - debug('length less than watermark', doRead); - } - - // however, if we've ended, then there's no point, and if we're already - // reading, then it's unnecessary. - if (state.ended || state.reading) { - doRead = false; - debug('reading or ended', doRead); - } else if (doRead) { - debug('do read'); - state.reading = true; - state.sync = true; - // if the length is currently zero, then we *need* a readable event. - if (state.length === 0) state.needReadable = true; - // call internal read method - this._read(state.highWaterMark); - state.sync = false; - // If _read pushed data synchronously, then `reading` will be false, - // and we need to re-evaluate how much data we can return to the user. - if (!state.reading) n = howMuchToRead(nOrig, state); - } - - var ret; - if (n > 0) ret = fromList(n, state);else ret = null; - - if (ret === null) { - state.needReadable = true; - n = 0; - } else { - state.length -= n; - } - - if (state.length === 0) { - // If we have nothing in the buffer, then we want to know - // as soon as we *do* get something into the buffer. - if (!state.ended) state.needReadable = true; - - // If we tried to read() past the EOF, then emit end on the next tick. - if (nOrig !== n && state.ended) endReadable(this); - } - - if (ret !== null) this.emit('data', ret); - - return ret; - }; - - function chunkInvalid(state, chunk) { - var er = null; - if (!isBuffer(chunk) && typeof chunk !== 'string' && chunk !== null && chunk !== undefined && !state.objectMode) { - er = new TypeError('Invalid non-string/buffer chunk'); - } - return er; - } - - function onEofChunk(stream, state) { - if (state.ended) return; - if (state.decoder) { - var chunk = state.decoder.end(); - if (chunk && chunk.length) { - state.buffer.push(chunk); - state.length += state.objectMode ? 1 : chunk.length; - } - } - state.ended = true; - - // emit 'readable' now to make sure it gets picked up. - emitReadable(stream); - } - - // Don't emit readable right away in sync mode, because this can trigger - // another read() call => stack overflow. This way, it might trigger - // a nextTick recursion warning, but that's not so bad. - function emitReadable(stream) { - var state = stream._readableState; - state.needReadable = false; - if (!state.emittedReadable) { - debug('emitReadable', state.flowing); - state.emittedReadable = true; - if (state.sync) nextTick(emitReadable_, stream);else emitReadable_(stream); - } - } - - function emitReadable_(stream) { - debug('emit readable'); - stream.emit('readable'); - flow(stream); - } - - // at this point, the user has presumably seen the 'readable' event, - // and called read() to consume some data. that may have triggered - // in turn another _read(n) call, in which case reading = true if - // it's in progress. - // However, if we're not ended, or reading, and the length < hwm, - // then go ahead and try to read some more preemptively. - function maybeReadMore(stream, state) { - if (!state.readingMore) { - state.readingMore = true; - nextTick(maybeReadMore_, stream, state); - } - } - - function maybeReadMore_(stream, state) { - var len = state.length; - while (!state.reading && !state.flowing && !state.ended && state.length < state.highWaterMark) { - debug('maybeReadMore read 0'); - stream.read(0); - if (len === state.length) - // didn't get any data, stop spinning. - break;else len = state.length; - } - state.readingMore = false; - } - - // abstract method. to be overridden in specific implementation classes. - // call cb(er, data) where data is <= n in length. - // for virtual (non-string, non-buffer) streams, "length" is somewhat - // arbitrary, and perhaps not very meaningful. - Readable.prototype._read = function (n) { - this.emit('error', new Error('not implemented')); - }; - - Readable.prototype.pipe = function (dest, pipeOpts) { - var src = this; - var state = this._readableState; - - switch (state.pipesCount) { - case 0: - state.pipes = dest; - break; - case 1: - state.pipes = [state.pipes, dest]; - break; - default: - state.pipes.push(dest); - break; - } - state.pipesCount += 1; - debug('pipe count=%d opts=%j', state.pipesCount, pipeOpts); - - var doEnd = (!pipeOpts || pipeOpts.end !== false); - - var endFn = doEnd ? onend : cleanup; - if (state.endEmitted) nextTick(endFn);else src.once('end', endFn); - - dest.on('unpipe', onunpipe); - function onunpipe(readable) { - debug('onunpipe'); - if (readable === src) { - cleanup(); - } - } - - function onend() { - debug('onend'); - dest.end(); - } - - // when the dest drains, it reduces the awaitDrain counter - // on the source. This would be more elegant with a .once() - // handler in flow(), but adding and removing repeatedly is - // too slow. - var ondrain = pipeOnDrain(src); - dest.on('drain', ondrain); - - var cleanedUp = false; - function cleanup() { - debug('cleanup'); - // cleanup event handlers once the pipe is broken - dest.removeListener('close', onclose); - dest.removeListener('finish', onfinish); - dest.removeListener('drain', ondrain); - dest.removeListener('error', onerror); - dest.removeListener('unpipe', onunpipe); - src.removeListener('end', onend); - src.removeListener('end', cleanup); - src.removeListener('data', ondata); - - cleanedUp = true; - - // if the reader is waiting for a drain event from this - // specific writer, then it would cause it to never start - // flowing again. - // So, if this is awaiting a drain, then we just call it now. - // If we don't know, then assume that we are waiting for one. - if (state.awaitDrain && (!dest._writableState || dest._writableState.needDrain)) ondrain(); - } - - // If the user pushes more data while we're writing to dest then we'll end up - // in ondata again. However, we only want to increase awaitDrain once because - // dest will only emit one 'drain' event for the multiple writes. - // => Introduce a guard on increasing awaitDrain. - var increasedAwaitDrain = false; - src.on('data', ondata); - function ondata(chunk) { - debug('ondata'); - increasedAwaitDrain = false; - var ret = dest.write(chunk); - if (false === ret && !increasedAwaitDrain) { - // If the user unpiped during `dest.write()`, it is possible - // to get stuck in a permanently paused state if that write - // also returned false. - // => Check whether `dest` is still a piping destination. - if ((state.pipesCount === 1 && state.pipes === dest || state.pipesCount > 1 && indexOf(state.pipes, dest) !== -1) && !cleanedUp) { - debug('false write response, pause', src._readableState.awaitDrain); - src._readableState.awaitDrain++; - increasedAwaitDrain = true; - } - src.pause(); - } - } - - // if the dest has an error, then stop piping into it. - // however, don't suppress the throwing behavior for this. - function onerror(er) { - debug('onerror', er); - unpipe(); - dest.removeListener('error', onerror); - if (listenerCount(dest, 'error') === 0) dest.emit('error', er); - } - - // Make sure our error handler is attached before userland ones. - prependListener(dest, 'error', onerror); - - // Both close and finish should trigger unpipe, but only once. - function onclose() { - dest.removeListener('finish', onfinish); - unpipe(); - } - dest.once('close', onclose); - function onfinish() { - debug('onfinish'); - dest.removeListener('close', onclose); - unpipe(); - } - dest.once('finish', onfinish); - - function unpipe() { - debug('unpipe'); - src.unpipe(dest); - } - - // tell the dest that it's being piped to - dest.emit('pipe', src); - - // start the flow if it hasn't been started already. - if (!state.flowing) { - debug('pipe resume'); - src.resume(); - } - - return dest; - }; - - function pipeOnDrain(src) { - return function () { - var state = src._readableState; - debug('pipeOnDrain', state.awaitDrain); - if (state.awaitDrain) state.awaitDrain--; - if (state.awaitDrain === 0 && src.listeners('data').length) { - state.flowing = true; - flow(src); - } - }; - } - - Readable.prototype.unpipe = function (dest) { - var state = this._readableState; - - // if we're not piping anywhere, then do nothing. - if (state.pipesCount === 0) return this; - - // just one destination. most common case. - if (state.pipesCount === 1) { - // passed in one, but it's not the right one. - if (dest && dest !== state.pipes) return this; - - if (!dest) dest = state.pipes; - - // got a match. - state.pipes = null; - state.pipesCount = 0; - state.flowing = false; - if (dest) dest.emit('unpipe', this); - return this; - } - - // slow case. multiple pipe destinations. - - if (!dest) { - // remove all. - var dests = state.pipes; - var len = state.pipesCount; - state.pipes = null; - state.pipesCount = 0; - state.flowing = false; - - for (var _i = 0; _i < len; _i++) { - dests[_i].emit('unpipe', this); - }return this; - } - - // try to find the right one. - var i = indexOf(state.pipes, dest); - if (i === -1) return this; - - state.pipes.splice(i, 1); - state.pipesCount -= 1; - if (state.pipesCount === 1) state.pipes = state.pipes[0]; - - dest.emit('unpipe', this); - - return this; - }; - - // set up data events if they are asked for - // Ensure readable listeners eventually get something - Readable.prototype.on = function (ev, fn) { - var res = EventEmitter.prototype.on.call(this, ev, fn); - - if (ev === 'data') { - // Start flowing on next tick if stream isn't explicitly paused - if (this._readableState.flowing !== false) this.resume(); - } else if (ev === 'readable') { - var state = this._readableState; - if (!state.endEmitted && !state.readableListening) { - state.readableListening = state.needReadable = true; - state.emittedReadable = false; - if (!state.reading) { - nextTick(nReadingNextTick, this); - } else if (state.length) { - emitReadable(this); - } - } - } - - return res; - }; - Readable.prototype.addListener = Readable.prototype.on; - - function nReadingNextTick(self) { - debug('readable nexttick read 0'); - self.read(0); - } - - // pause() and resume() are remnants of the legacy readable stream API - // If the user uses them, then switch into old mode. - Readable.prototype.resume = function () { - var state = this._readableState; - if (!state.flowing) { - debug('resume'); - state.flowing = true; - resume(this, state); - } - return this; - }; - - function resume(stream, state) { - if (!state.resumeScheduled) { - state.resumeScheduled = true; - nextTick(resume_, stream, state); - } - } - - function resume_(stream, state) { - if (!state.reading) { - debug('resume read 0'); - stream.read(0); - } - - state.resumeScheduled = false; - state.awaitDrain = 0; - stream.emit('resume'); - flow(stream); - if (state.flowing && !state.reading) stream.read(0); - } - - Readable.prototype.pause = function () { - debug('call pause flowing=%j', this._readableState.flowing); - if (false !== this._readableState.flowing) { - debug('pause'); - this._readableState.flowing = false; - this.emit('pause'); - } - return this; - }; - - function flow(stream) { - var state = stream._readableState; - debug('flow', state.flowing); - while (state.flowing && stream.read() !== null) {} - } - - // wrap an old-style stream as the async data source. - // This is *not* part of the readable stream interface. - // It is an ugly unfortunate mess of history. - Readable.prototype.wrap = function (stream) { - var state = this._readableState; - var paused = false; - - var self = this; - stream.on('end', function () { - debug('wrapped end'); - if (state.decoder && !state.ended) { - var chunk = state.decoder.end(); - if (chunk && chunk.length) self.push(chunk); - } - - self.push(null); - }); - - stream.on('data', function (chunk) { - debug('wrapped data'); - if (state.decoder) chunk = state.decoder.write(chunk); - - // don't skip over falsy values in objectMode - if (state.objectMode && (chunk === null || chunk === undefined)) return;else if (!state.objectMode && (!chunk || !chunk.length)) return; - - var ret = self.push(chunk); - if (!ret) { - paused = true; - stream.pause(); - } - }); - - // proxy all the other methods. - // important when wrapping filters and duplexes. - for (var i in stream) { - if (this[i] === undefined && typeof stream[i] === 'function') { - this[i] = function (method) { - return function () { - return stream[method].apply(stream, arguments); - }; - }(i); - } - } - - // proxy certain important events. - var events = ['error', 'close', 'destroy', 'pause', 'resume']; - forEach(events, function (ev) { - stream.on(ev, self.emit.bind(self, ev)); - }); - - // when we try to consume some more bytes, simply unpause the - // underlying stream. - self._read = function (n) { - debug('wrapped _read', n); - if (paused) { - paused = false; - stream.resume(); - } - }; - - return self; - }; - - // exposed for testing purposes only. - Readable._fromList = fromList; - - // Pluck off n bytes from an array of buffers. - // Length is the combined lengths of all the buffers in the list. - // This function is designed to be inlinable, so please take care when making - // changes to the function body. - function fromList(n, state) { - // nothing buffered - if (state.length === 0) return null; - - var ret; - if (state.objectMode) ret = state.buffer.shift();else if (!n || n >= state.length) { - // read it all, truncate the list - if (state.decoder) ret = state.buffer.join('');else if (state.buffer.length === 1) ret = state.buffer.head.data;else ret = state.buffer.concat(state.length); - state.buffer.clear(); - } else { - // read part of list - ret = fromListPartial(n, state.buffer, state.decoder); - } - - return ret; - } - - // Extracts only enough buffered data to satisfy the amount requested. - // This function is designed to be inlinable, so please take care when making - // changes to the function body. - function fromListPartial(n, list, hasStrings) { - var ret; - if (n < list.head.data.length) { - // slice is the same for buffers and strings - ret = list.head.data.slice(0, n); - list.head.data = list.head.data.slice(n); - } else if (n === list.head.data.length) { - // first chunk is a perfect match - ret = list.shift(); - } else { - // result spans more than one buffer - ret = hasStrings ? copyFromBufferString(n, list) : copyFromBuffer(n, list); - } - return ret; - } - - // Copies a specified amount of characters from the list of buffered data - // chunks. - // This function is designed to be inlinable, so please take care when making - // changes to the function body. - function copyFromBufferString(n, list) { - var p = list.head; - var c = 1; - var ret = p.data; - n -= ret.length; - while (p = p.next) { - var str = p.data; - var nb = n > str.length ? str.length : n; - if (nb === str.length) ret += str;else ret += str.slice(0, n); - n -= nb; - if (n === 0) { - if (nb === str.length) { - ++c; - if (p.next) list.head = p.next;else list.head = list.tail = null; - } else { - list.head = p; - p.data = str.slice(nb); - } - break; - } - ++c; - } - list.length -= c; - return ret; - } - - // Copies a specified amount of bytes from the list of buffered data chunks. - // This function is designed to be inlinable, so please take care when making - // changes to the function body. - function copyFromBuffer(n, list) { - var ret = Buffer.allocUnsafe(n); - var p = list.head; - var c = 1; - p.data.copy(ret); - n -= p.data.length; - while (p = p.next) { - var buf = p.data; - var nb = n > buf.length ? buf.length : n; - buf.copy(ret, ret.length - n, 0, nb); - n -= nb; - if (n === 0) { - if (nb === buf.length) { - ++c; - if (p.next) list.head = p.next;else list.head = list.tail = null; - } else { - list.head = p; - p.data = buf.slice(nb); - } - break; - } - ++c; - } - list.length -= c; - return ret; - } - - function endReadable(stream) { - var state = stream._readableState; - - // If we get here before consuming all the bytes, then that is a - // bug in node. Should never happen. - if (state.length > 0) throw new Error('"endReadable()" called on non-empty stream'); - - if (!state.endEmitted) { - state.ended = true; - nextTick(endReadableNT, state, stream); - } - } - - function endReadableNT(state, stream) { - // Check that we didn't get one last unshift. - if (!state.endEmitted && state.length === 0) { - state.endEmitted = true; - stream.readable = false; - stream.emit('end'); - } - } - - function forEach(xs, f) { - for (var i = 0, l = xs.length; i < l; i++) { - f(xs[i], i); - } - } - - function indexOf(xs, x) { - for (var i = 0, l = xs.length; i < l; i++) { - if (xs[i] === x) return i; - } - return -1; - } - - // A bit simpler than readable streams. - Writable.WritableState = WritableState; - inherits$1(Writable, EventEmitter); - - function nop() {} - - function WriteReq(chunk, encoding, cb) { - this.chunk = chunk; - this.encoding = encoding; - this.callback = cb; - this.next = null; - } - - function WritableState(options, stream) { - Object.defineProperty(this, 'buffer', { - get: deprecate(function () { - return this.getBuffer(); - }, '_writableState.buffer is deprecated. Use _writableState.getBuffer ' + 'instead.') - }); - options = options || {}; - - // object stream flag to indicate whether or not this stream - // contains buffers or objects. - this.objectMode = !!options.objectMode; - - if (stream instanceof Duplex) this.objectMode = this.objectMode || !!options.writableObjectMode; - - // the point at which write() starts returning false - // Note: 0 is a valid value, means that we always return false if - // the entire buffer is not flushed immediately on write() - var hwm = options.highWaterMark; - var defaultHwm = this.objectMode ? 16 : 16 * 1024; - this.highWaterMark = hwm || hwm === 0 ? hwm : defaultHwm; - - // cast to ints. - this.highWaterMark = ~ ~this.highWaterMark; - - this.needDrain = false; - // at the start of calling end() - this.ending = false; - // when end() has been called, and returned - this.ended = false; - // when 'finish' is emitted - this.finished = false; - - // should we decode strings into buffers before passing to _write? - // this is here so that some node-core streams can optimize string - // handling at a lower level. - var noDecode = options.decodeStrings === false; - this.decodeStrings = !noDecode; - - // Crypto is kind of old and crusty. Historically, its default string - // encoding is 'binary' so we have to make this configurable. - // Everything else in the universe uses 'utf8', though. - this.defaultEncoding = options.defaultEncoding || 'utf8'; - - // not an actual buffer we keep track of, but a measurement - // of how much we're waiting to get pushed to some underlying - // socket or file. - this.length = 0; - - // a flag to see when we're in the middle of a write. - this.writing = false; - - // when true all writes will be buffered until .uncork() call - this.corked = 0; - - // a flag to be able to tell if the onwrite cb is called immediately, - // or on a later tick. We set this to true at first, because any - // actions that shouldn't happen until "later" should generally also - // not happen before the first write call. - this.sync = true; - - // a flag to know if we're processing previously buffered items, which - // may call the _write() callback in the same tick, so that we don't - // end up in an overlapped onwrite situation. - this.bufferProcessing = false; - - // the callback that's passed to _write(chunk,cb) - this.onwrite = function (er) { - onwrite(stream, er); - }; - - // the callback that the user supplies to write(chunk,encoding,cb) - this.writecb = null; - - // the amount that is being written when _write is called. - this.writelen = 0; - - this.bufferedRequest = null; - this.lastBufferedRequest = null; - - // number of pending user-supplied write callbacks - // this must be 0 before 'finish' can be emitted - this.pendingcb = 0; - - // emit prefinish if the only thing we're waiting for is _write cbs - // This is relevant for synchronous Transform streams - this.prefinished = false; - - // True if the error was already emitted and should not be thrown again - this.errorEmitted = false; - - // count buffered requests - this.bufferedRequestCount = 0; - - // allocate the first CorkedRequest, there is always - // one allocated and free to use, and we maintain at most two - this.corkedRequestsFree = new CorkedRequest(this); - } - - WritableState.prototype.getBuffer = function writableStateGetBuffer() { - var current = this.bufferedRequest; - var out = []; - while (current) { - out.push(current); - current = current.next; - } - return out; - }; - function Writable(options) { - - // Writable ctor is applied to Duplexes, though they're not - // instanceof Writable, they're instanceof Readable. - if (!(this instanceof Writable) && !(this instanceof Duplex)) return new Writable(options); - - this._writableState = new WritableState(options, this); - - // legacy. - this.writable = true; - - if (options) { - if (typeof options.write === 'function') this._write = options.write; - - if (typeof options.writev === 'function') this._writev = options.writev; - } - - EventEmitter.call(this); - } - - // Otherwise people can pipe Writable streams, which is just wrong. - Writable.prototype.pipe = function () { - this.emit('error', new Error('Cannot pipe, not readable')); - }; - - function writeAfterEnd(stream, cb) { - var er = new Error('write after end'); - // TODO: defer error events consistently everywhere, not just the cb - stream.emit('error', er); - nextTick(cb, er); - } - - // If we get something that is not a buffer, string, null, or undefined, - // and we're not in objectMode, then that's an error. - // Otherwise stream chunks are all considered to be of length=1, and the - // watermarks determine how many objects to keep in the buffer, rather than - // how many bytes or characters. - function validChunk(stream, state, chunk, cb) { - var valid = true; - var er = false; - // Always throw error if a null is written - // if we are not in object mode then throw - // if it is not a buffer, string, or undefined. - if (chunk === null) { - er = new TypeError('May not write null values to stream'); - } else if (!Buffer.isBuffer(chunk) && typeof chunk !== 'string' && chunk !== undefined && !state.objectMode) { - er = new TypeError('Invalid non-string/buffer chunk'); - } - if (er) { - stream.emit('error', er); - nextTick(cb, er); - valid = false; - } - return valid; - } - - Writable.prototype.write = function (chunk, encoding, cb) { - var state = this._writableState; - var ret = false; - - if (typeof encoding === 'function') { - cb = encoding; - encoding = null; - } - - if (Buffer.isBuffer(chunk)) encoding = 'buffer';else if (!encoding) encoding = state.defaultEncoding; - - if (typeof cb !== 'function') cb = nop; - - if (state.ended) writeAfterEnd(this, cb);else if (validChunk(this, state, chunk, cb)) { - state.pendingcb++; - ret = writeOrBuffer(this, state, chunk, encoding, cb); - } - - return ret; - }; - - Writable.prototype.cork = function () { - var state = this._writableState; - - state.corked++; - }; - - Writable.prototype.uncork = function () { - var state = this._writableState; - - if (state.corked) { - state.corked--; - - if (!state.writing && !state.corked && !state.finished && !state.bufferProcessing && state.bufferedRequest) clearBuffer(this, state); - } - }; - - Writable.prototype.setDefaultEncoding = function setDefaultEncoding(encoding) { - // node::ParseEncoding() requires lower case. - if (typeof encoding === 'string') encoding = encoding.toLowerCase(); - if (!(['hex', 'utf8', 'utf-8', 'ascii', 'binary', 'base64', 'ucs2', 'ucs-2', 'utf16le', 'utf-16le', 'raw'].indexOf((encoding + '').toLowerCase()) > -1)) throw new TypeError('Unknown encoding: ' + encoding); - this._writableState.defaultEncoding = encoding; + 'use strict'; + + var domain; + + // This constructor is used to store event handlers. Instantiating this is + // faster than explicitly calling `Object.create(null)` to get a "clean" empty + // object (tested with v8 v4.9). + function EventHandlers() {} + EventHandlers.prototype = Object.create(null); + + function EventEmitter() { + EventEmitter.init.call(this); + } + + // nodejs oddity + // require('events') === require('events').EventEmitter + EventEmitter.EventEmitter = EventEmitter; + + EventEmitter.usingDomains = false; + + EventEmitter.prototype.domain = undefined; + EventEmitter.prototype._events = undefined; + EventEmitter.prototype._maxListeners = undefined; + + // By default EventEmitters will print a warning if more than 10 listeners are + // added to it. This is a useful default which helps finding memory leaks. + EventEmitter.defaultMaxListeners = 10; + + EventEmitter.init = function() { + this.domain = null; + if (EventEmitter.usingDomains) { + // if there is an active domain, then attach to it. + if (domain.active ) ; + } + + if (!this._events || this._events === Object.getPrototypeOf(this)._events) { + this._events = new EventHandlers(); + this._eventsCount = 0; + } + + this._maxListeners = this._maxListeners || undefined; + }; + + // Obviously not all Emitters should be limited to 10. This function allows + // that to be increased. Set to zero for unlimited. + EventEmitter.prototype.setMaxListeners = function setMaxListeners(n) { + if (typeof n !== 'number' || n < 0 || isNaN(n)) + throw new TypeError('"n" argument must be a positive number'); + this._maxListeners = n; + return this; + }; + + function $getMaxListeners(that) { + if (that._maxListeners === undefined) + return EventEmitter.defaultMaxListeners; + return that._maxListeners; + } + + EventEmitter.prototype.getMaxListeners = function getMaxListeners() { + return $getMaxListeners(this); + }; + + // These standalone emit* functions are used to optimize calling of event + // handlers for fast cases because emit() itself often has a variable number of + // arguments and can be deoptimized because of that. These functions always have + // the same number of arguments and thus do not get deoptimized, so the code + // inside them can execute faster. + function emitNone(handler, isFn, self) { + if (isFn) + handler.call(self); + else { + var len = handler.length; + var listeners = arrayClone(handler, len); + for (var i = 0; i < len; ++i) + listeners[i].call(self); + } + } + function emitOne(handler, isFn, self, arg1) { + if (isFn) + handler.call(self, arg1); + else { + var len = handler.length; + var listeners = arrayClone(handler, len); + for (var i = 0; i < len; ++i) + listeners[i].call(self, arg1); + } + } + function emitTwo(handler, isFn, self, arg1, arg2) { + if (isFn) + handler.call(self, arg1, arg2); + else { + var len = handler.length; + var listeners = arrayClone(handler, len); + for (var i = 0; i < len; ++i) + listeners[i].call(self, arg1, arg2); + } + } + function emitThree(handler, isFn, self, arg1, arg2, arg3) { + if (isFn) + handler.call(self, arg1, arg2, arg3); + else { + var len = handler.length; + var listeners = arrayClone(handler, len); + for (var i = 0; i < len; ++i) + listeners[i].call(self, arg1, arg2, arg3); + } + } + + function emitMany(handler, isFn, self, args) { + if (isFn) + handler.apply(self, args); + else { + var len = handler.length; + var listeners = arrayClone(handler, len); + for (var i = 0; i < len; ++i) + listeners[i].apply(self, args); + } + } + + EventEmitter.prototype.emit = function emit(type) { + var er, handler, len, args, i, events, domain; + var doError = (type === 'error'); + + events = this._events; + if (events) + doError = (doError && events.error == null); + else if (!doError) + return false; + + domain = this.domain; + + // If there is no 'error' event listener then throw. + if (doError) { + er = arguments[1]; + if (domain) { + if (!er) + er = new Error('Uncaught, unspecified "error" event'); + er.domainEmitter = this; + er.domain = domain; + er.domainThrown = false; + domain.emit('error', er); + } else if (er instanceof Error) { + throw er; // Unhandled 'error' event + } else { + // At least give some kind of context to the user + var err = new Error('Uncaught, unspecified "error" event. (' + er + ')'); + err.context = er; + throw err; + } + return false; + } + + handler = events[type]; + + if (!handler) + return false; + + var isFn = typeof handler === 'function'; + len = arguments.length; + switch (len) { + // fast cases + case 1: + emitNone(handler, isFn, this); + break; + case 2: + emitOne(handler, isFn, this, arguments[1]); + break; + case 3: + emitTwo(handler, isFn, this, arguments[1], arguments[2]); + break; + case 4: + emitThree(handler, isFn, this, arguments[1], arguments[2], arguments[3]); + break; + // slower + default: + args = new Array(len - 1); + for (i = 1; i < len; i++) + args[i - 1] = arguments[i]; + emitMany(handler, isFn, this, args); + } + + return true; + }; + + function _addListener(target, type, listener, prepend) { + var m; + var events; + var existing; + + if (typeof listener !== 'function') + throw new TypeError('"listener" argument must be a function'); + + events = target._events; + if (!events) { + events = target._events = new EventHandlers(); + target._eventsCount = 0; + } else { + // To avoid recursion in the case that type === "newListener"! Before + // adding it to the listeners, first emit "newListener". + if (events.newListener) { + target.emit('newListener', type, + listener.listener ? listener.listener : listener); + + // Re-assign `events` because a newListener handler could have caused the + // this._events to be assigned to a new object + events = target._events; + } + existing = events[type]; + } + + if (!existing) { + // Optimize the case of one listener. Don't need the extra array object. + existing = events[type] = listener; + ++target._eventsCount; + } else { + if (typeof existing === 'function') { + // Adding the second element, need to change to array. + existing = events[type] = prepend ? [listener, existing] : + [existing, listener]; + } else { + // If we've already got an array, just append. + if (prepend) { + existing.unshift(listener); + } else { + existing.push(listener); + } + } + + // Check for listener leak + if (!existing.warned) { + m = $getMaxListeners(target); + if (m && m > 0 && existing.length > m) { + existing.warned = true; + var w = new Error('Possible EventEmitter memory leak detected. ' + + existing.length + ' ' + type + ' listeners added. ' + + 'Use emitter.setMaxListeners() to increase limit'); + w.name = 'MaxListenersExceededWarning'; + w.emitter = target; + w.type = type; + w.count = existing.length; + emitWarning(w); + } + } + } + + return target; + } + function emitWarning(e) { + typeof console.warn === 'function' ? console.warn(e) : console.log(e); + } + EventEmitter.prototype.addListener = function addListener(type, listener) { + return _addListener(this, type, listener, false); + }; + + EventEmitter.prototype.on = EventEmitter.prototype.addListener; + + EventEmitter.prototype.prependListener = + function prependListener(type, listener) { + return _addListener(this, type, listener, true); + }; + + function _onceWrap(target, type, listener) { + var fired = false; + function g() { + target.removeListener(type, g); + if (!fired) { + fired = true; + listener.apply(target, arguments); + } + } + g.listener = listener; + return g; + } + + EventEmitter.prototype.once = function once(type, listener) { + if (typeof listener !== 'function') + throw new TypeError('"listener" argument must be a function'); + this.on(type, _onceWrap(this, type, listener)); + return this; + }; + + EventEmitter.prototype.prependOnceListener = + function prependOnceListener(type, listener) { + if (typeof listener !== 'function') + throw new TypeError('"listener" argument must be a function'); + this.prependListener(type, _onceWrap(this, type, listener)); + return this; + }; + + // emits a 'removeListener' event iff the listener was removed + EventEmitter.prototype.removeListener = + function removeListener(type, listener) { + var list, events, position, i, originalListener; + + if (typeof listener !== 'function') + throw new TypeError('"listener" argument must be a function'); + + events = this._events; + if (!events) + return this; + + list = events[type]; + if (!list) + return this; + + if (list === listener || (list.listener && list.listener === listener)) { + if (--this._eventsCount === 0) + this._events = new EventHandlers(); + else { + delete events[type]; + if (events.removeListener) + this.emit('removeListener', type, list.listener || listener); + } + } else if (typeof list !== 'function') { + position = -1; + + for (i = list.length; i-- > 0;) { + if (list[i] === listener || + (list[i].listener && list[i].listener === listener)) { + originalListener = list[i].listener; + position = i; + break; + } + } + + if (position < 0) + return this; + + if (list.length === 1) { + list[0] = undefined; + if (--this._eventsCount === 0) { + this._events = new EventHandlers(); return this; - }; - - function decodeChunk(state, chunk, encoding) { - if (!state.objectMode && state.decodeStrings !== false && typeof chunk === 'string') { - chunk = Buffer.from(chunk, encoding); - } - return chunk; - } - - // if we're already writing something, then just put this - // in the queue, and wait our turn. Otherwise, call _write - // If we return false, then we need a drain event, so set that flag. - function writeOrBuffer(stream, state, chunk, encoding, cb) { - chunk = decodeChunk(state, chunk, encoding); - - if (Buffer.isBuffer(chunk)) encoding = 'buffer'; - var len = state.objectMode ? 1 : chunk.length; - - state.length += len; - - var ret = state.length < state.highWaterMark; - // we must ensure that previous needDrain will not be reset to false. - if (!ret) state.needDrain = true; - - if (state.writing || state.corked) { - var last = state.lastBufferedRequest; - state.lastBufferedRequest = new WriteReq(chunk, encoding, cb); - if (last) { - last.next = state.lastBufferedRequest; - } else { - state.bufferedRequest = state.lastBufferedRequest; - } - state.bufferedRequestCount += 1; - } else { - doWrite(stream, state, false, len, chunk, encoding, cb); - } - - return ret; - } - - function doWrite(stream, state, writev, len, chunk, encoding, cb) { - state.writelen = len; - state.writecb = cb; - state.writing = true; - state.sync = true; - if (writev) stream._writev(chunk, state.onwrite);else stream._write(chunk, encoding, state.onwrite); - state.sync = false; - } - - function onwriteError(stream, state, sync, er, cb) { - --state.pendingcb; - if (sync) nextTick(cb, er);else cb(er); - - stream._writableState.errorEmitted = true; - stream.emit('error', er); - } - - function onwriteStateUpdate(state) { - state.writing = false; - state.writecb = null; - state.length -= state.writelen; - state.writelen = 0; - } - - function onwrite(stream, er) { - var state = stream._writableState; - var sync = state.sync; - var cb = state.writecb; - - onwriteStateUpdate(state); - - if (er) onwriteError(stream, state, sync, er, cb);else { - // Check if we're actually ready to finish, but don't emit yet - var finished = needFinish(state); - - if (!finished && !state.corked && !state.bufferProcessing && state.bufferedRequest) { - clearBuffer(stream, state); - } - - if (sync) { - /**/ - nextTick(afterWrite, stream, state, finished, cb); - /**/ - } else { - afterWrite(stream, state, finished, cb); - } - } - } - - function afterWrite(stream, state, finished, cb) { - if (!finished) onwriteDrain(stream, state); - state.pendingcb--; - cb(); - finishMaybe(stream, state); - } - - // Must force callback to be called on nextTick, so that we don't - // emit 'drain' before the write() consumer gets the 'false' return - // value, and has a chance to attach a 'drain' listener. - function onwriteDrain(stream, state) { - if (state.length === 0 && state.needDrain) { - state.needDrain = false; - stream.emit('drain'); - } - } - - // if there's something in the buffer waiting, then process it - function clearBuffer(stream, state) { - state.bufferProcessing = true; - var entry = state.bufferedRequest; - - if (stream._writev && entry && entry.next) { - // Fast case, write everything using _writev() - var l = state.bufferedRequestCount; - var buffer = new Array(l); - var holder = state.corkedRequestsFree; - holder.entry = entry; - - var count = 0; - while (entry) { - buffer[count] = entry; - entry = entry.next; - count += 1; - } - - doWrite(stream, state, true, state.length, buffer, '', holder.finish); - - // doWrite is almost always async, defer these to save a bit of time - // as the hot path ends with doWrite - state.pendingcb++; - state.lastBufferedRequest = null; - if (holder.next) { - state.corkedRequestsFree = holder.next; - holder.next = null; - } else { - state.corkedRequestsFree = new CorkedRequest(state); - } - } else { - // Slow case, write chunks one-by-one - while (entry) { - var chunk = entry.chunk; - var encoding = entry.encoding; - var cb = entry.callback; - var len = state.objectMode ? 1 : chunk.length; - - doWrite(stream, state, false, len, chunk, encoding, cb); - entry = entry.next; - // if we didn't call the onwrite immediately, then - // it means that we need to wait until it does. - // also, that means that the chunk and cb are currently - // being processed, so move the buffer counter past them. - if (state.writing) { - break; - } - } - - if (entry === null) state.lastBufferedRequest = null; - } - - state.bufferedRequestCount = 0; - state.bufferedRequest = entry; - state.bufferProcessing = false; - } - - Writable.prototype._write = function (chunk, encoding, cb) { - cb(new Error('not implemented')); - }; - - Writable.prototype._writev = null; - - Writable.prototype.end = function (chunk, encoding, cb) { - var state = this._writableState; - - if (typeof chunk === 'function') { - cb = chunk; - chunk = null; - encoding = null; - } else if (typeof encoding === 'function') { - cb = encoding; - encoding = null; - } - - if (chunk !== null && chunk !== undefined) this.write(chunk, encoding); - - // .end() fully uncorks - if (state.corked) { - state.corked = 1; - this.uncork(); - } - - // ignore unnecessary end() calls. - if (!state.ending && !state.finished) endWritable(this, state, cb); - }; - - function needFinish(state) { - return state.ending && state.length === 0 && state.bufferedRequest === null && !state.finished && !state.writing; - } - - function prefinish(stream, state) { - if (!state.prefinished) { - state.prefinished = true; - stream.emit('prefinish'); - } - } - - function finishMaybe(stream, state) { - var need = needFinish(state); - if (need) { - if (state.pendingcb === 0) { - prefinish(stream, state); - state.finished = true; - stream.emit('finish'); - } else { - prefinish(stream, state); - } - } - return need; - } - - function endWritable(stream, state, cb) { - state.ending = true; - finishMaybe(stream, state); - if (cb) { - if (state.finished) nextTick(cb);else stream.once('finish', cb); - } - state.ended = true; - stream.writable = false; - } - - // It seems a linked list but it is not - // there will be only 2 of these for each stream - function CorkedRequest(state) { - var _this = this; - - this.next = null; - this.entry = null; - - this.finish = function (err) { - var entry = _this.entry; - _this.entry = null; - while (entry) { - var cb = entry.callback; - state.pendingcb--; - cb(err); - entry = entry.next; - } - if (state.corkedRequestsFree) { - state.corkedRequestsFree.next = _this; - } else { - state.corkedRequestsFree = _this; - } - }; - } - - inherits$1(Duplex, Readable); - - var keys = Object.keys(Writable.prototype); - for (var v = 0; v < keys.length; v++) { - var method = keys[v]; - if (!Duplex.prototype[method]) Duplex.prototype[method] = Writable.prototype[method]; - } - function Duplex(options) { - if (!(this instanceof Duplex)) return new Duplex(options); - - Readable.call(this, options); - Writable.call(this, options); - - if (options && options.readable === false) this.readable = false; - - if (options && options.writable === false) this.writable = false; - - this.allowHalfOpen = true; - if (options && options.allowHalfOpen === false) this.allowHalfOpen = false; - - this.once('end', onend); - } - - // the no-half-open enforcer - function onend() { - // if we allow half-open state, or if the writable side ended, - // then we're ok. - if (this.allowHalfOpen || this._writableState.ended) return; - - // no more data can be written. - // But allow more writes to happen in this tick. - nextTick(onEndNT, this); - } - - function onEndNT(self) { - self.end(); - } - - // a transform stream is a readable/writable stream where you do - inherits$1(Transform, Duplex); - - function TransformState(stream) { - this.afterTransform = function (er, data) { - return afterTransform(stream, er, data); - }; - - this.needTransform = false; - this.transforming = false; - this.writecb = null; - this.writechunk = null; - this.writeencoding = null; - } - - function afterTransform(stream, er, data) { - var ts = stream._transformState; - ts.transforming = false; - - var cb = ts.writecb; - - if (!cb) return stream.emit('error', new Error('no writecb in Transform class')); - - ts.writechunk = null; - ts.writecb = null; - - if (data !== null && data !== undefined) stream.push(data); - - cb(er); - - var rs = stream._readableState; - rs.reading = false; - if (rs.needReadable || rs.length < rs.highWaterMark) { - stream._read(rs.highWaterMark); - } - } - function Transform(options) { - if (!(this instanceof Transform)) return new Transform(options); - - Duplex.call(this, options); - - this._transformState = new TransformState(this); - - // when the writable side finishes, then flush out anything remaining. - var stream = this; - - // start out asking for a readable event once data is transformed. - this._readableState.needReadable = true; - - // we have implemented the _read method, and done the other things - // that Readable wants before the first _read call, so unset the - // sync guard flag. - this._readableState.sync = false; - - if (options) { - if (typeof options.transform === 'function') this._transform = options.transform; - - if (typeof options.flush === 'function') this._flush = options.flush; - } - - this.once('prefinish', function () { - if (typeof this._flush === 'function') this._flush(function (er) { - done(stream, er); - });else done(stream); - }); - } - - Transform.prototype.push = function (chunk, encoding) { - this._transformState.needTransform = false; - return Duplex.prototype.push.call(this, chunk, encoding); - }; - - // This is the part where you do stuff! - // override this function in implementation classes. - // 'chunk' is an input chunk. - // - // Call `push(newChunk)` to pass along transformed output - // to the readable side. You may call 'push' zero or more times. - // - // Call `cb(err)` when you are done with this chunk. If you pass - // an error, then that'll put the hurt on the whole operation. If you - // never call cb(), then you'll never get another chunk. - Transform.prototype._transform = function (chunk, encoding, cb) { - throw new Error('Not implemented'); - }; - - Transform.prototype._write = function (chunk, encoding, cb) { - var ts = this._transformState; - ts.writecb = cb; - ts.writechunk = chunk; - ts.writeencoding = encoding; - if (!ts.transforming) { - var rs = this._readableState; - if (ts.needTransform || rs.needReadable || rs.length < rs.highWaterMark) this._read(rs.highWaterMark); - } - }; - - // Doesn't matter what the args are here. - // _transform does all the work. - // That we got here means that the readable side wants more data. - Transform.prototype._read = function (n) { - var ts = this._transformState; - - if (ts.writechunk !== null && ts.writecb && !ts.transforming) { - ts.transforming = true; - this._transform(ts.writechunk, ts.writeencoding, ts.afterTransform); - } else { - // mark that we need a transform, so that any data that comes in - // will get processed, now that we've asked for it. - ts.needTransform = true; - } - }; - - function done(stream, er) { - if (er) return stream.emit('error', er); - - // if there's nothing in the write buffer, then that means - // that nothing more will ever be provided - var ws = stream._writableState; - var ts = stream._transformState; - - if (ws.length) throw new Error('Calling transform done when ws.length != 0'); - - if (ts.transforming) throw new Error('Calling transform done when still transforming'); - - return stream.push(null); - } - - inherits$1(PassThrough, Transform); - function PassThrough(options) { - if (!(this instanceof PassThrough)) return new PassThrough(options); - - Transform.call(this, options); - } - - PassThrough.prototype._transform = function (chunk, encoding, cb) { - cb(null, chunk); - }; - - inherits$1(Stream, EventEmitter); - Stream.Readable = Readable; - Stream.Writable = Writable; - Stream.Duplex = Duplex; - Stream.Transform = Transform; - Stream.PassThrough = PassThrough; - - // Backwards-compat with node 0.4.x - Stream.Stream = Stream; - - // old-style streams. Note that the pipe method (the only relevant - // part of this class) is overridden in the Readable class. - - function Stream() { - EventEmitter.call(this); - } - - Stream.prototype.pipe = function(dest, options) { - var source = this; - - function ondata(chunk) { - if (dest.writable) { - if (false === dest.write(chunk) && source.pause) { - source.pause(); - } - } - } - - source.on('data', ondata); - - function ondrain() { - if (source.readable && source.resume) { - source.resume(); - } - } - - dest.on('drain', ondrain); - - // If the 'end' option is not supplied, dest.end() will be called when - // source gets the 'end' or 'close' events. Only dest.end() once. - if (!dest._isStdio && (!options || options.end !== false)) { - source.on('end', onend); - source.on('close', onclose); - } - - var didOnEnd = false; - function onend() { - if (didOnEnd) return; - didOnEnd = true; - - dest.end(); - } - - - function onclose() { - if (didOnEnd) return; - didOnEnd = true; - - if (typeof dest.destroy === 'function') dest.destroy(); - } - - // don't leave dangling pipes when there are errors. - function onerror(er) { - cleanup(); - if (EventEmitter.listenerCount(this, 'error') === 0) { - throw er; // Unhandled stream error in pipe. - } - } - - source.on('error', onerror); - dest.on('error', onerror); - - // remove all the event listeners that were added. - function cleanup() { - source.removeListener('data', ondata); - dest.removeListener('drain', ondrain); - - source.removeListener('end', onend); - source.removeListener('close', onclose); - - source.removeListener('error', onerror); - dest.removeListener('error', onerror); - - source.removeListener('end', cleanup); - source.removeListener('close', cleanup); - - dest.removeListener('close', cleanup); - } - - source.on('end', cleanup); - source.on('close', cleanup); - - dest.on('close', cleanup); - - dest.emit('pipe', source); - - // Allow for unix-like usage: A.pipe(B).pipe(C) - return dest; - }; - - const bom_utf8 = Buffer.from([239, 187, 191]); - - class CsvError extends Error { - constructor(code, message, ...contexts) { - if(Array.isArray(message)) message = message.join(' '); - super(message); - if(Error.captureStackTrace !== undefined){ - Error.captureStackTrace(this, CsvError); - } - this.code = code; - for(const context of contexts){ - for(const key in context){ - const value = context[key]; - this[key] = isBuffer(value) ? value.toString() : value == null ? value : JSON.parse(JSON.stringify(value)); - } - } - } - } - - const isObject = function(obj){ - return typeof obj === 'object' && obj !== null && ! Array.isArray(obj); - }; - - const underscore = function(str){ - return str.replace(/([A-Z])/g, function(_, match){ - return '_' + match.toLowerCase(); - }); - }; - - // Lodash implementation of `get` - - const charCodeOfDot = '.'.charCodeAt(0); - const reEscapeChar = /\\(\\)?/g; - const rePropName = RegExp( - // Match anything that isn't a dot or bracket. - '[^.[\\]]+' + '|' + - // Or match property names within brackets. - '\\[(?:' + - // Match a non-string expression. - '([^"\'][^[]*)' + '|' + - // Or match strings (supports escaping characters). - '(["\'])((?:(?!\\2)[^\\\\]|\\\\.)*?)\\2' + - ')\\]'+ '|' + - // Or match "" as the space between consecutive dots or empty brackets. - '(?=(?:\\.|\\[\\])(?:\\.|\\[\\]|$))' - , 'g'); - const reIsDeepProp = /\.|\[(?:[^[\]]*|(["'])(?:(?!\1)[^\\]|\\.)*?\1)\]/; - const reIsPlainProp = /^\w*$/; - const getTag = function(value){ - return Object.prototype.toString.call(value); - }; - const isSymbol = function(value){ - const type = typeof value; - return type === 'symbol' || (type === 'object' && value && getTag(value) === '[object Symbol]'); - }; - const isKey = function(value, object){ - if(Array.isArray(value)){ - return false; - } - const type = typeof value; - if(type === 'number' || type === 'symbol' || type === 'boolean' || !value || isSymbol(value)){ - return true; - } - return reIsPlainProp.test(value) || !reIsDeepProp.test(value) || - (object != null && value in Object(object)); - }; - const stringToPath = function(string){ - const result = []; - if(string.charCodeAt(0) === charCodeOfDot){ - result.push(''); - } - string.replace(rePropName, function(match, expression, quote, subString){ - let key = match; - if(quote){ - key = subString.replace(reEscapeChar, '$1'); - }else if(expression){ - key = expression.trim(); - } - result.push(key); - }); - return result; - }; - const castPath = function(value, object){ - if(Array.isArray(value)){ - return value; - } else { - return isKey(value, object) ? [value] : stringToPath(value); - } - }; - const toKey = function(value){ - if(typeof value === 'string' || isSymbol(value)) - return value; - const result = `${value}`; - // eslint-disable-next-line - return (result == '0' && (1 / value) == -INFINITY) ? '-0' : result; - }; - const get = function(object, path){ - path = castPath(path, object); - let index = 0; - const length = path.length; - while(object != null && index < length){ - object = object[toKey(path[index++])]; - } - return (index && index === length) ? object : undefined; - }; - - class Stringifier extends Transform { - constructor(opts = {}){ - super({...{writableObjectMode: true}, ...opts}); - const options = {}; - let err; - // Merge with user options - for(const opt in opts){ - options[underscore(opt)] = opts[opt]; - } - if((err = this.normalize(options)) !== undefined) throw err; - switch(options.record_delimiter){ - case 'auto': - options.record_delimiter = null; - break; - case 'unix': - options.record_delimiter = "\n"; - break; - case 'mac': - options.record_delimiter = "\r"; - break; - case 'windows': - options.record_delimiter = "\r\n"; - break; - case 'ascii': - options.record_delimiter = "\u001e"; - break; - case 'unicode': - options.record_delimiter = "\u2028"; - break; - } - // Expose options - this.options = options; - // Internal state - this.state = { - stop: false - }; - // Information - this.info = { - records: 0 - }; - } - normalize(options){ - // Normalize option `bom` - if(options.bom === undefined || options.bom === null || options.bom === false){ - options.bom = false; - }else if(options.bom !== true){ - return new CsvError('CSV_OPTION_BOOLEAN_INVALID_TYPE', [ - 'option `bom` is optional and must be a boolean value,', - `got ${JSON.stringify(options.bom)}` - ]); - } - // Normalize option `delimiter` - if(options.delimiter === undefined || options.delimiter === null){ - options.delimiter = ','; - }else if(isBuffer(options.delimiter)){ - options.delimiter = options.delimiter.toString(); - }else if(typeof options.delimiter !== 'string'){ - return new CsvError('CSV_OPTION_DELIMITER_INVALID_TYPE', [ - 'option `delimiter` must be a buffer or a string,', - `got ${JSON.stringify(options.delimiter)}` - ]); - } - // Normalize option `quote` - if(options.quote === undefined || options.quote === null){ - options.quote = '"'; - }else if(options.quote === true){ - options.quote = '"'; - }else if(options.quote === false){ - options.quote = ''; - }else if (isBuffer(options.quote)){ - options.quote = options.quote.toString(); - }else if(typeof options.quote !== 'string'){ - return new CsvError('CSV_OPTION_QUOTE_INVALID_TYPE', [ - 'option `quote` must be a boolean, a buffer or a string,', - `got ${JSON.stringify(options.quote)}` - ]); - } - // Normalize option `quoted` - if(options.quoted === undefined || options.quoted === null){ - options.quoted = false; - } - // Normalize option `quoted_empty` - if(options.quoted_empty === undefined || options.quoted_empty === null){ - options.quoted_empty = undefined; - } - // Normalize option `quoted_match` - if(options.quoted_match === undefined || options.quoted_match === null || options.quoted_match === false){ - options.quoted_match = null; - }else if(!Array.isArray(options.quoted_match)){ - options.quoted_match = [options.quoted_match]; - } - if(options.quoted_match){ - for(const quoted_match of options.quoted_match){ - const isString = typeof quoted_match === 'string'; - const isRegExp = quoted_match instanceof RegExp; - if(!isString && !isRegExp){ - return Error(`Invalid Option: quoted_match must be a string or a regex, got ${JSON.stringify(quoted_match)}`); - } - } - } - // Normalize option `quoted_string` - if(options.quoted_string === undefined || options.quoted_string === null){ - options.quoted_string = false; - } - // Normalize option `eof` - if(options.eof === undefined || options.eof === null){ - options.eof = true; - } - // Normalize option `escape` - if(options.escape === undefined || options.escape === null){ - options.escape = '"'; - }else if(isBuffer(options.escape)){ - options.escape = options.escape.toString(); - }else if(typeof options.escape !== 'string'){ - return Error(`Invalid Option: escape must be a buffer or a string, got ${JSON.stringify(options.escape)}`); - } - if (options.escape.length > 1){ - return Error(`Invalid Option: escape must be one character, got ${options.escape.length} characters`); - } - // Normalize option `header` - if(options.header === undefined || options.header === null){ - options.header = false; - } - // Normalize option `columns` - const [err, columns] = this.normalize_columns(options.columns); - if(err) return err; - options.columns = columns; - // Normalize option `quoted` - if(options.quoted === undefined || options.quoted === null){ - options.quoted = false; - } - // Normalize option `cast` - if(options.cast === undefined || options.cast === null){ - options.cast = {}; - } - // Normalize option cast.bigint - if(options.cast.bigint === undefined || options.cast.bigint === null){ - // Cast boolean to string by default - options.cast.bigint = value => '' + value; - } - // Normalize option cast.boolean - if(options.cast.boolean === undefined || options.cast.boolean === null){ - // Cast boolean to string by default - options.cast.boolean = value => value ? '1' : ''; - } - // Normalize option cast.date - if(options.cast.date === undefined || options.cast.date === null){ - // Cast date to timestamp string by default - options.cast.date = value => '' + value.getTime(); - } - // Normalize option cast.number - if(options.cast.number === undefined || options.cast.number === null){ - // Cast number to string using native casting by default - options.cast.number = value => '' + value; - } - // Normalize option cast.object - if(options.cast.object === undefined || options.cast.object === null){ - // Stringify object as JSON by default - options.cast.object = value => JSON.stringify(value); - } - // Normalize option cast.string - if(options.cast.string === undefined || options.cast.string === null){ - // Leave string untouched - options.cast.string = function(value){return value;}; - } - // Normalize option `record_delimiter` - if(options.record_delimiter === undefined || options.record_delimiter === null){ - options.record_delimiter = '\n'; - }else if(isBuffer(options.record_delimiter)){ - options.record_delimiter = options.record_delimiter.toString(); - }else if(typeof options.record_delimiter !== 'string'){ - return Error(`Invalid Option: record_delimiter must be a buffer or a string, got ${JSON.stringify(options.record_delimiter)}`); - } - } - _transform(chunk, encoding, callback){ - if(this.state.stop === true){ - return; - } - const err = this.__transform(chunk); - if(err !== undefined){ - this.state.stop = true; - } - callback(err); - } - _flush(callback){ - if(this.state.stop === true){ - // Note, Node.js 12 call flush even after an error, we must prevent - // `callback` from being called in flush without any error. - return; - } - if(this.info.records === 0){ - this.bom(); - const err = this.headers(); - if(err) callback(err); - } - callback(); - } - __transform(chunk){ - // Chunk validation - if(!Array.isArray(chunk) && typeof chunk !== 'object'){ - return Error(`Invalid Record: expect an array or an object, got ${JSON.stringify(chunk)}`); - } - // Detect columns from the first record - if(this.info.records === 0){ - if(Array.isArray(chunk)){ - if(this.options.header === true && this.options.columns === undefined){ - return Error('Undiscoverable Columns: header option requires column option or object records'); - } - }else if(this.options.columns === undefined){ - const [err, columns] = this.normalize_columns(Object.keys(chunk)); - if(err) return; - this.options.columns = columns; - } - } - // Emit the header - if(this.info.records === 0){ - this.bom(); - const err = this.headers(); - if(err) return err; - } - // Emit and stringify the record if an object or an array - try{ - this.emit('record', chunk, this.info.records); - }catch(err){ - return err; - } - // Convert the record into a string - let err, chunk_string; - if(this.options.eof){ - [err, chunk_string] = this.stringify(chunk); - if(err) return err; - if(chunk_string === undefined){ - return; - }else { - chunk_string = chunk_string + this.options.record_delimiter; - } - }else { - [err, chunk_string] = this.stringify(chunk); - if(err) return err; - if(chunk_string === undefined){ - return; - }else { - if(this.options.header || this.info.records){ - chunk_string = this.options.record_delimiter + chunk_string; - } - } - } - // Emit the csv - this.info.records++; - this.push(chunk_string); - } - stringify(chunk, chunkIsHeader=false){ - if(typeof chunk !== 'object'){ - return [undefined, chunk]; - } - const {columns} = this.options; - const record = []; - // Record is an array - if(Array.isArray(chunk)){ - // We are getting an array but the user has specified output columns. In - // this case, we respect the columns indexes - if(columns){ - chunk.splice(columns.length); - } - // Cast record elements - for(let i=0; i= 0; - const containsQuote = (quote !== '') && value.indexOf(quote) >= 0; - const containsEscape = value.indexOf(escape) >= 0 && (escape !== quote); - const containsRecordDelimiter = value.indexOf(record_delimiter) >= 0; - const quotedString = quoted_string && typeof field === 'string'; - let quotedMatch = quoted_match && quoted_match.filter(quoted_match => { - if(typeof quoted_match === 'string'){ - return value.indexOf(quoted_match) !== -1; - }else { - return quoted_match.test(value); - } - }); - quotedMatch = quotedMatch && quotedMatch.length > 0; - const shouldQuote = containsQuote === true || containsdelimiter || containsRecordDelimiter || quoted || quotedString || quotedMatch; - if(shouldQuote === true && containsEscape === true){ - const regexp = escape === '\\' - ? new RegExp(escape + escape, 'g') - : new RegExp(escape, 'g'); - value = value.replace(regexp, escape + escape); - } - if(containsQuote === true){ - const regexp = new RegExp(quote,'g'); - value = value.replace(regexp, escape + quote); - } - if(shouldQuote === true){ - value = quote + value + quote; - } - csvrecord += value; - }else if(quoted_empty === true || (field === '' && quoted_string === true && quoted_empty !== false)){ - csvrecord += quote + quote; - } - if(i !== record.length - 1){ - csvrecord += delimiter; - } - } - return [undefined, csvrecord]; - } - bom(){ - if(this.options.bom !== true){ - return; - } - this.push(bom_utf8); - } - headers(){ - if(this.options.header === false){ - return; - } - if(this.options.columns === undefined){ - return; - } - let err; - let headers = this.options.columns.map(column => column.header); - if(this.options.eof){ - [err, headers] = this.stringify(headers, true); - headers += this.options.record_delimiter; - }else { - [err, headers] = this.stringify(headers); - } - if(err) return err; - this.push(headers); - } - __cast(value, context){ - const type = typeof value; - try{ - if(type === 'string'){ // Fine for 99% of the cases - return [undefined, this.options.cast.string(value, context)]; - }else if(type === 'bigint'){ - return [undefined, this.options.cast.bigint(value, context)]; - }else if(type === 'number'){ - return [undefined, this.options.cast.number(value, context)]; - }else if(type === 'boolean'){ - return [undefined, this.options.cast.boolean(value, context)]; - }else if(value instanceof Date){ - return [undefined, this.options.cast.date(value, context)]; - }else if(type === 'object' && value !== null){ - return [undefined, this.options.cast.object(value, context)]; - }else { - return [undefined, value, value]; - } - }catch(err){ - return [err]; - } - } - normalize_columns(columns){ - if(columns === undefined || columns === null){ - return []; - } - if(typeof columns !== 'object'){ - return [Error('Invalid option "columns": expect an array or an object')]; - } - if(!Array.isArray(columns)){ - const newcolumns = []; - for(const k in columns){ - newcolumns.push({ - key: k, - header: columns[k] - }); - } - columns = newcolumns; - }else { - const newcolumns = []; - for(const column of columns){ - if(typeof column === 'string'){ - newcolumns.push({ - key: column, - header: column - }); - }else if(typeof column === 'object' && column !== undefined && !Array.isArray(column)){ - if(!column.key){ - return [Error('Invalid column definition: property "key" is required')]; - } - if(column.header === undefined){ - column.header = column.key; - } - newcolumns.push(column); - }else { - return [Error('Invalid column definition: expect a string or an object')]; - } - } - columns = newcolumns; - } - return [undefined, columns]; - } - } - - const stringify = function(){ - let data, options, callback; - for(const i in arguments){ - const argument = arguments[i]; - const type = typeof argument; - if(data === undefined && (Array.isArray(argument))){ - data = argument; - }else if(options === undefined && isObject(argument)){ - options = argument; - }else if(callback === undefined && type === 'function'){ - callback = argument; - }else { - throw new CsvError('CSV_INVALID_ARGUMENT', [ - 'Invalid argument:', - `got ${JSON.stringify(argument)} at index ${i}` - ]); - } - } - const stringifier = new Stringifier(options); - if(callback){ - const chunks = []; - stringifier.on('readable', function(){ - let chunk; - while((chunk = this.read()) !== null){ - chunks.push(chunk); - } - }); - stringifier.on('error', function(err){ - callback(err); - }); - stringifier.on('end', function(){ - callback(undefined, chunks.join('')); - }); - } - if(data !== undefined){ - const writer = function(){ - for(const record of data){ - stringifier.write(record); - } - stringifier.end(); - }; - // Support Deno, Rollup doesnt provide a shim for setImmediate - if(typeof setImmediate === 'function'){ - setImmediate(writer); - }else { - setTimeout(writer, 0); - } - } - return stringifier; - }; - - exports.CsvError = CsvError; - exports.Stringifier = Stringifier; - exports.stringify = stringify; - - Object.defineProperty(exports, '__esModule', { value: true }); - - return exports; + } else { + delete events[type]; + } + } else { + spliceOne(list, position); + } + + if (events.removeListener) + this.emit('removeListener', type, originalListener || listener); + } + + return this; + }; + + EventEmitter.prototype.removeAllListeners = + function removeAllListeners(type) { + var listeners, events; + + events = this._events; + if (!events) + return this; + + // not listening for removeListener, no need to emit + if (!events.removeListener) { + if (arguments.length === 0) { + this._events = new EventHandlers(); + this._eventsCount = 0; + } else if (events[type]) { + if (--this._eventsCount === 0) + this._events = new EventHandlers(); + else + delete events[type]; + } + return this; + } + + // emit removeListener for all listeners on all events + if (arguments.length === 0) { + var keys = Object.keys(events); + for (var i = 0, key; i < keys.length; ++i) { + key = keys[i]; + if (key === 'removeListener') continue; + this.removeAllListeners(key); + } + this.removeAllListeners('removeListener'); + this._events = new EventHandlers(); + this._eventsCount = 0; + return this; + } + + listeners = events[type]; + + if (typeof listeners === 'function') { + this.removeListener(type, listeners); + } else if (listeners) { + // LIFO order + do { + this.removeListener(type, listeners[listeners.length - 1]); + } while (listeners[0]); + } + + return this; + }; + + EventEmitter.prototype.listeners = function listeners(type) { + var evlistener; + var ret; + var events = this._events; + + if (!events) + ret = []; + else { + evlistener = events[type]; + if (!evlistener) + ret = []; + else if (typeof evlistener === 'function') + ret = [evlistener.listener || evlistener]; + else + ret = unwrapListeners(evlistener); + } + + return ret; + }; + + EventEmitter.listenerCount = function(emitter, type) { + if (typeof emitter.listenerCount === 'function') { + return emitter.listenerCount(type); + } else { + return listenerCount$1.call(emitter, type); + } + }; + + EventEmitter.prototype.listenerCount = listenerCount$1; + function listenerCount$1(type) { + var events = this._events; + + if (events) { + var evlistener = events[type]; + + if (typeof evlistener === 'function') { + return 1; + } else if (evlistener) { + return evlistener.length; + } + } + + return 0; + } + + EventEmitter.prototype.eventNames = function eventNames() { + return this._eventsCount > 0 ? Reflect.ownKeys(this._events) : []; + }; + + // About 1.5x faster than the two-arg version of Array#splice(). + function spliceOne(list, index) { + for (var i = index, k = i + 1, n = list.length; k < n; i += 1, k += 1) + list[i] = list[k]; + list.pop(); + } + + function arrayClone(arr, i) { + var copy = new Array(i); + while (i--) + copy[i] = arr[i]; + return copy; + } + + function unwrapListeners(arr) { + var ret = new Array(arr.length); + for (var i = 0; i < ret.length; ++i) { + ret[i] = arr[i].listener || arr[i]; + } + return ret; + } + + var global$1 = (typeof global !== "undefined" ? global : + typeof self !== "undefined" ? self : + typeof window !== "undefined" ? window : {}); + + var lookup = []; + var revLookup = []; + var Arr = typeof Uint8Array !== 'undefined' ? Uint8Array : Array; + var inited = false; + function init () { + inited = true; + var code = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'; + for (var i = 0, len = code.length; i < len; ++i) { + lookup[i] = code[i]; + revLookup[code.charCodeAt(i)] = i; + } + + revLookup['-'.charCodeAt(0)] = 62; + revLookup['_'.charCodeAt(0)] = 63; + } + + function toByteArray (b64) { + if (!inited) { + init(); + } + var i, j, l, tmp, placeHolders, arr; + var len = b64.length; + + if (len % 4 > 0) { + throw new Error('Invalid string. Length must be a multiple of 4') + } + + // the number of equal signs (place holders) + // if there are two placeholders, than the two characters before it + // represent one byte + // if there is only one, then the three characters before it represent 2 bytes + // this is just a cheap hack to not do indexOf twice + placeHolders = b64[len - 2] === '=' ? 2 : b64[len - 1] === '=' ? 1 : 0; + + // base64 is 4/3 + up to two characters of the original data + arr = new Arr(len * 3 / 4 - placeHolders); + + // if there are placeholders, only get up to the last complete 4 chars + l = placeHolders > 0 ? len - 4 : len; + + var L = 0; + + for (i = 0, j = 0; i < l; i += 4, j += 3) { + tmp = (revLookup[b64.charCodeAt(i)] << 18) | (revLookup[b64.charCodeAt(i + 1)] << 12) | (revLookup[b64.charCodeAt(i + 2)] << 6) | revLookup[b64.charCodeAt(i + 3)]; + arr[L++] = (tmp >> 16) & 0xFF; + arr[L++] = (tmp >> 8) & 0xFF; + arr[L++] = tmp & 0xFF; + } + + if (placeHolders === 2) { + tmp = (revLookup[b64.charCodeAt(i)] << 2) | (revLookup[b64.charCodeAt(i + 1)] >> 4); + arr[L++] = tmp & 0xFF; + } else if (placeHolders === 1) { + tmp = (revLookup[b64.charCodeAt(i)] << 10) | (revLookup[b64.charCodeAt(i + 1)] << 4) | (revLookup[b64.charCodeAt(i + 2)] >> 2); + arr[L++] = (tmp >> 8) & 0xFF; + arr[L++] = tmp & 0xFF; + } + + return arr + } + + function tripletToBase64 (num) { + return lookup[num >> 18 & 0x3F] + lookup[num >> 12 & 0x3F] + lookup[num >> 6 & 0x3F] + lookup[num & 0x3F] + } + + function encodeChunk (uint8, start, end) { + var tmp; + var output = []; + for (var i = start; i < end; i += 3) { + tmp = (uint8[i] << 16) + (uint8[i + 1] << 8) + (uint8[i + 2]); + output.push(tripletToBase64(tmp)); + } + return output.join('') + } + + function fromByteArray (uint8) { + if (!inited) { + init(); + } + var tmp; + var len = uint8.length; + var extraBytes = len % 3; // if we have 1 byte left, pad 2 bytes + var output = ''; + var parts = []; + var maxChunkLength = 16383; // must be multiple of 3 + + // go through the array every three bytes, we'll deal with trailing stuff later + for (var i = 0, len2 = len - extraBytes; i < len2; i += maxChunkLength) { + parts.push(encodeChunk(uint8, i, (i + maxChunkLength) > len2 ? len2 : (i + maxChunkLength))); + } + + // pad the end with zeros, but make sure to not forget the extra bytes + if (extraBytes === 1) { + tmp = uint8[len - 1]; + output += lookup[tmp >> 2]; + output += lookup[(tmp << 4) & 0x3F]; + output += '=='; + } else if (extraBytes === 2) { + tmp = (uint8[len - 2] << 8) + (uint8[len - 1]); + output += lookup[tmp >> 10]; + output += lookup[(tmp >> 4) & 0x3F]; + output += lookup[(tmp << 2) & 0x3F]; + output += '='; + } + + parts.push(output); + + return parts.join('') + } + + function read (buffer, offset, isLE, mLen, nBytes) { + var e, m; + var eLen = nBytes * 8 - mLen - 1; + var eMax = (1 << eLen) - 1; + var eBias = eMax >> 1; + var nBits = -7; + var i = isLE ? (nBytes - 1) : 0; + var d = isLE ? -1 : 1; + var s = buffer[offset + i]; + + i += d; + + e = s & ((1 << (-nBits)) - 1); + s >>= (-nBits); + nBits += eLen; + for (; nBits > 0; e = e * 256 + buffer[offset + i], i += d, nBits -= 8) {} + + m = e & ((1 << (-nBits)) - 1); + e >>= (-nBits); + nBits += mLen; + for (; nBits > 0; m = m * 256 + buffer[offset + i], i += d, nBits -= 8) {} + + if (e === 0) { + e = 1 - eBias; + } else if (e === eMax) { + return m ? NaN : ((s ? -1 : 1) * Infinity) + } else { + m = m + Math.pow(2, mLen); + e = e - eBias; + } + return (s ? -1 : 1) * m * Math.pow(2, e - mLen) + } + + function write (buffer, value, offset, isLE, mLen, nBytes) { + var e, m, c; + var eLen = nBytes * 8 - mLen - 1; + var eMax = (1 << eLen) - 1; + var eBias = eMax >> 1; + var rt = (mLen === 23 ? Math.pow(2, -24) - Math.pow(2, -77) : 0); + var i = isLE ? 0 : (nBytes - 1); + var d = isLE ? 1 : -1; + var s = value < 0 || (value === 0 && 1 / value < 0) ? 1 : 0; + + value = Math.abs(value); + + if (isNaN(value) || value === Infinity) { + m = isNaN(value) ? 1 : 0; + e = eMax; + } else { + e = Math.floor(Math.log(value) / Math.LN2); + if (value * (c = Math.pow(2, -e)) < 1) { + e--; + c *= 2; + } + if (e + eBias >= 1) { + value += rt / c; + } else { + value += rt * Math.pow(2, 1 - eBias); + } + if (value * c >= 2) { + e++; + c /= 2; + } + + if (e + eBias >= eMax) { + m = 0; + e = eMax; + } else if (e + eBias >= 1) { + m = (value * c - 1) * Math.pow(2, mLen); + e = e + eBias; + } else { + m = value * Math.pow(2, eBias - 1) * Math.pow(2, mLen); + e = 0; + } + } + + for (; mLen >= 8; buffer[offset + i] = m & 0xff, i += d, m /= 256, mLen -= 8) {} + + e = (e << mLen) | m; + eLen += mLen; + for (; eLen > 0; buffer[offset + i] = e & 0xff, i += d, e /= 256, eLen -= 8) {} + + buffer[offset + i - d] |= s * 128; + } + + var toString = {}.toString; + + var isArray$1 = Array.isArray || function (arr) { + return toString.call(arr) == '[object Array]'; + }; + + var INSPECT_MAX_BYTES = 50; + + /** + * If `Buffer.TYPED_ARRAY_SUPPORT`: + * === true Use Uint8Array implementation (fastest) + * === false Use Object implementation (most compatible, even IE6) + * + * Browsers that support typed arrays are IE 10+, Firefox 4+, Chrome 7+, Safari 5.1+, + * Opera 11.6+, iOS 4.2+. + * + * Due to various browser bugs, sometimes the Object implementation will be used even + * when the browser supports typed arrays. + * + * Note: + * + * - Firefox 4-29 lacks support for adding new properties to `Uint8Array` instances, + * See: https://bugzilla.mozilla.org/show_bug.cgi?id=695438. + * + * - Chrome 9-10 is missing the `TypedArray.prototype.subarray` function. + * + * - IE10 has a broken `TypedArray.prototype.subarray` function which returns arrays of + * incorrect length in some situations. + + * We detect these buggy browsers and set `Buffer.TYPED_ARRAY_SUPPORT` to `false` so they + * get the Object implementation, which is slower but behaves correctly. + */ + Buffer.TYPED_ARRAY_SUPPORT = global$1.TYPED_ARRAY_SUPPORT !== undefined + ? global$1.TYPED_ARRAY_SUPPORT + : true; + + function kMaxLength () { + return Buffer.TYPED_ARRAY_SUPPORT + ? 0x7fffffff + : 0x3fffffff + } + + function createBuffer (that, length) { + if (kMaxLength() < length) { + throw new RangeError('Invalid typed array length') + } + if (Buffer.TYPED_ARRAY_SUPPORT) { + // Return an augmented `Uint8Array` instance, for best performance + that = new Uint8Array(length); + that.__proto__ = Buffer.prototype; + } else { + // Fallback: Return an object instance of the Buffer class + if (that === null) { + that = new Buffer(length); + } + that.length = length; + } + + return that + } + + /** + * The Buffer constructor returns instances of `Uint8Array` that have their + * prototype changed to `Buffer.prototype`. Furthermore, `Buffer` is a subclass of + * `Uint8Array`, so the returned instances will have all the node `Buffer` methods + * and the `Uint8Array` methods. Square bracket notation works as expected -- it + * returns a single octet. + * + * The `Uint8Array` prototype remains unmodified. + */ + + function Buffer (arg, encodingOrOffset, length) { + if (!Buffer.TYPED_ARRAY_SUPPORT && !(this instanceof Buffer)) { + return new Buffer(arg, encodingOrOffset, length) + } + + // Common case. + if (typeof arg === 'number') { + if (typeof encodingOrOffset === 'string') { + throw new Error( + 'If encoding is specified then the first argument must be a string' + ) + } + return allocUnsafe(this, arg) + } + return from(this, arg, encodingOrOffset, length) + } + + Buffer.poolSize = 8192; // not used by this implementation + + // TODO: Legacy, not needed anymore. Remove in next major version. + Buffer._augment = function (arr) { + arr.__proto__ = Buffer.prototype; + return arr + }; + + function from (that, value, encodingOrOffset, length) { + if (typeof value === 'number') { + throw new TypeError('"value" argument must not be a number') + } + + if (typeof ArrayBuffer !== 'undefined' && value instanceof ArrayBuffer) { + return fromArrayBuffer(that, value, encodingOrOffset, length) + } + + if (typeof value === 'string') { + return fromString(that, value, encodingOrOffset) + } + + return fromObject(that, value) + } + + /** + * Functionally equivalent to Buffer(arg, encoding) but throws a TypeError + * if value is a number. + * Buffer.from(str[, encoding]) + * Buffer.from(array) + * Buffer.from(buffer) + * Buffer.from(arrayBuffer[, byteOffset[, length]]) + **/ + Buffer.from = function (value, encodingOrOffset, length) { + return from(null, value, encodingOrOffset, length) + }; + + if (Buffer.TYPED_ARRAY_SUPPORT) { + Buffer.prototype.__proto__ = Uint8Array.prototype; + Buffer.__proto__ = Uint8Array; + } + + function assertSize (size) { + if (typeof size !== 'number') { + throw new TypeError('"size" argument must be a number') + } else if (size < 0) { + throw new RangeError('"size" argument must not be negative') + } + } + + function alloc (that, size, fill, encoding) { + assertSize(size); + if (size <= 0) { + return createBuffer(that, size) + } + if (fill !== undefined) { + // Only pay attention to encoding if it's a string. This + // prevents accidentally sending in a number that would + // be interpretted as a start offset. + return typeof encoding === 'string' + ? createBuffer(that, size).fill(fill, encoding) + : createBuffer(that, size).fill(fill) + } + return createBuffer(that, size) + } + + /** + * Creates a new filled Buffer instance. + * alloc(size[, fill[, encoding]]) + **/ + Buffer.alloc = function (size, fill, encoding) { + return alloc(null, size, fill, encoding) + }; + + function allocUnsafe (that, size) { + assertSize(size); + that = createBuffer(that, size < 0 ? 0 : checked(size) | 0); + if (!Buffer.TYPED_ARRAY_SUPPORT) { + for (var i = 0; i < size; ++i) { + that[i] = 0; + } + } + return that + } + + /** + * Equivalent to Buffer(num), by default creates a non-zero-filled Buffer instance. + * */ + Buffer.allocUnsafe = function (size) { + return allocUnsafe(null, size) + }; + /** + * Equivalent to SlowBuffer(num), by default creates a non-zero-filled Buffer instance. + */ + Buffer.allocUnsafeSlow = function (size) { + return allocUnsafe(null, size) + }; + + function fromString (that, string, encoding) { + if (typeof encoding !== 'string' || encoding === '') { + encoding = 'utf8'; + } + + if (!Buffer.isEncoding(encoding)) { + throw new TypeError('"encoding" must be a valid string encoding') + } + + var length = byteLength(string, encoding) | 0; + that = createBuffer(that, length); + + var actual = that.write(string, encoding); + + if (actual !== length) { + // Writing a hex string, for example, that contains invalid characters will + // cause everything after the first invalid character to be ignored. (e.g. + // 'abxxcd' will be treated as 'ab') + that = that.slice(0, actual); + } + + return that + } + + function fromArrayLike (that, array) { + var length = array.length < 0 ? 0 : checked(array.length) | 0; + that = createBuffer(that, length); + for (var i = 0; i < length; i += 1) { + that[i] = array[i] & 255; + } + return that + } + + function fromArrayBuffer (that, array, byteOffset, length) { + array.byteLength; // this throws if `array` is not a valid ArrayBuffer + + if (byteOffset < 0 || array.byteLength < byteOffset) { + throw new RangeError('\'offset\' is out of bounds') + } + + if (array.byteLength < byteOffset + (length || 0)) { + throw new RangeError('\'length\' is out of bounds') + } + + if (byteOffset === undefined && length === undefined) { + array = new Uint8Array(array); + } else if (length === undefined) { + array = new Uint8Array(array, byteOffset); + } else { + array = new Uint8Array(array, byteOffset, length); + } + + if (Buffer.TYPED_ARRAY_SUPPORT) { + // Return an augmented `Uint8Array` instance, for best performance + that = array; + that.__proto__ = Buffer.prototype; + } else { + // Fallback: Return an object instance of the Buffer class + that = fromArrayLike(that, array); + } + return that + } + + function fromObject (that, obj) { + if (internalIsBuffer(obj)) { + var len = checked(obj.length) | 0; + that = createBuffer(that, len); + + if (that.length === 0) { + return that + } + + obj.copy(that, 0, 0, len); + return that + } + + if (obj) { + if ((typeof ArrayBuffer !== 'undefined' && + obj.buffer instanceof ArrayBuffer) || 'length' in obj) { + if (typeof obj.length !== 'number' || isnan(obj.length)) { + return createBuffer(that, 0) + } + return fromArrayLike(that, obj) + } + + if (obj.type === 'Buffer' && isArray$1(obj.data)) { + return fromArrayLike(that, obj.data) + } + } + + throw new TypeError('First argument must be a string, Buffer, ArrayBuffer, Array, or array-like object.') + } + + function checked (length) { + // Note: cannot use `length < kMaxLength()` here because that fails when + // length is NaN (which is otherwise coerced to zero.) + if (length >= kMaxLength()) { + throw new RangeError('Attempt to allocate Buffer larger than maximum ' + + 'size: 0x' + kMaxLength().toString(16) + ' bytes') + } + return length | 0 + } + Buffer.isBuffer = isBuffer; + function internalIsBuffer (b) { + return !!(b != null && b._isBuffer) + } + + Buffer.compare = function compare (a, b) { + if (!internalIsBuffer(a) || !internalIsBuffer(b)) { + throw new TypeError('Arguments must be Buffers') + } + + if (a === b) return 0 + + var x = a.length; + var y = b.length; + + for (var i = 0, len = Math.min(x, y); i < len; ++i) { + if (a[i] !== b[i]) { + x = a[i]; + y = b[i]; + break + } + } + + if (x < y) return -1 + if (y < x) return 1 + return 0 + }; + + Buffer.isEncoding = function isEncoding (encoding) { + switch (String(encoding).toLowerCase()) { + case 'hex': + case 'utf8': + case 'utf-8': + case 'ascii': + case 'latin1': + case 'binary': + case 'base64': + case 'ucs2': + case 'ucs-2': + case 'utf16le': + case 'utf-16le': + return true + default: + return false + } + }; + + Buffer.concat = function concat (list, length) { + if (!isArray$1(list)) { + throw new TypeError('"list" argument must be an Array of Buffers') + } + + if (list.length === 0) { + return Buffer.alloc(0) + } + + var i; + if (length === undefined) { + length = 0; + for (i = 0; i < list.length; ++i) { + length += list[i].length; + } + } + + var buffer = Buffer.allocUnsafe(length); + var pos = 0; + for (i = 0; i < list.length; ++i) { + var buf = list[i]; + if (!internalIsBuffer(buf)) { + throw new TypeError('"list" argument must be an Array of Buffers') + } + buf.copy(buffer, pos); + pos += buf.length; + } + return buffer + }; + + function byteLength (string, encoding) { + if (internalIsBuffer(string)) { + return string.length + } + if (typeof ArrayBuffer !== 'undefined' && typeof ArrayBuffer.isView === 'function' && + (ArrayBuffer.isView(string) || string instanceof ArrayBuffer)) { + return string.byteLength + } + if (typeof string !== 'string') { + string = '' + string; + } + + var len = string.length; + if (len === 0) return 0 + + // Use a for loop to avoid recursion + var loweredCase = false; + for (;;) { + switch (encoding) { + case 'ascii': + case 'latin1': + case 'binary': + return len + case 'utf8': + case 'utf-8': + case undefined: + return utf8ToBytes(string).length + case 'ucs2': + case 'ucs-2': + case 'utf16le': + case 'utf-16le': + return len * 2 + case 'hex': + return len >>> 1 + case 'base64': + return base64ToBytes(string).length + default: + if (loweredCase) return utf8ToBytes(string).length // assume utf8 + encoding = ('' + encoding).toLowerCase(); + loweredCase = true; + } + } + } + Buffer.byteLength = byteLength; + + function slowToString (encoding, start, end) { + var loweredCase = false; + + // No need to verify that "this.length <= MAX_UINT32" since it's a read-only + // property of a typed array. + + // This behaves neither like String nor Uint8Array in that we set start/end + // to their upper/lower bounds if the value passed is out of range. + // undefined is handled specially as per ECMA-262 6th Edition, + // Section 13.3.3.7 Runtime Semantics: KeyedBindingInitialization. + if (start === undefined || start < 0) { + start = 0; + } + // Return early if start > this.length. Done here to prevent potential uint32 + // coercion fail below. + if (start > this.length) { + return '' + } + + if (end === undefined || end > this.length) { + end = this.length; + } + + if (end <= 0) { + return '' + } + + // Force coersion to uint32. This will also coerce falsey/NaN values to 0. + end >>>= 0; + start >>>= 0; + + if (end <= start) { + return '' + } + + if (!encoding) encoding = 'utf8'; + + while (true) { + switch (encoding) { + case 'hex': + return hexSlice(this, start, end) + + case 'utf8': + case 'utf-8': + return utf8Slice(this, start, end) + + case 'ascii': + return asciiSlice(this, start, end) + + case 'latin1': + case 'binary': + return latin1Slice(this, start, end) + + case 'base64': + return base64Slice(this, start, end) + + case 'ucs2': + case 'ucs-2': + case 'utf16le': + case 'utf-16le': + return utf16leSlice(this, start, end) + + default: + if (loweredCase) throw new TypeError('Unknown encoding: ' + encoding) + encoding = (encoding + '').toLowerCase(); + loweredCase = true; + } + } + } + + // The property is used by `Buffer.isBuffer` and `is-buffer` (in Safari 5-7) to detect + // Buffer instances. + Buffer.prototype._isBuffer = true; + + function swap (b, n, m) { + var i = b[n]; + b[n] = b[m]; + b[m] = i; + } + + Buffer.prototype.swap16 = function swap16 () { + var len = this.length; + if (len % 2 !== 0) { + throw new RangeError('Buffer size must be a multiple of 16-bits') + } + for (var i = 0; i < len; i += 2) { + swap(this, i, i + 1); + } + return this + }; + + Buffer.prototype.swap32 = function swap32 () { + var len = this.length; + if (len % 4 !== 0) { + throw new RangeError('Buffer size must be a multiple of 32-bits') + } + for (var i = 0; i < len; i += 4) { + swap(this, i, i + 3); + swap(this, i + 1, i + 2); + } + return this + }; + + Buffer.prototype.swap64 = function swap64 () { + var len = this.length; + if (len % 8 !== 0) { + throw new RangeError('Buffer size must be a multiple of 64-bits') + } + for (var i = 0; i < len; i += 8) { + swap(this, i, i + 7); + swap(this, i + 1, i + 6); + swap(this, i + 2, i + 5); + swap(this, i + 3, i + 4); + } + return this + }; + + Buffer.prototype.toString = function toString () { + var length = this.length | 0; + if (length === 0) return '' + if (arguments.length === 0) return utf8Slice(this, 0, length) + return slowToString.apply(this, arguments) + }; + + Buffer.prototype.equals = function equals (b) { + if (!internalIsBuffer(b)) throw new TypeError('Argument must be a Buffer') + if (this === b) return true + return Buffer.compare(this, b) === 0 + }; + + Buffer.prototype.inspect = function inspect () { + var str = ''; + var max = INSPECT_MAX_BYTES; + if (this.length > 0) { + str = this.toString('hex', 0, max).match(/.{2}/g).join(' '); + if (this.length > max) str += ' ... '; + } + return '' + }; + + Buffer.prototype.compare = function compare (target, start, end, thisStart, thisEnd) { + if (!internalIsBuffer(target)) { + throw new TypeError('Argument must be a Buffer') + } + + if (start === undefined) { + start = 0; + } + if (end === undefined) { + end = target ? target.length : 0; + } + if (thisStart === undefined) { + thisStart = 0; + } + if (thisEnd === undefined) { + thisEnd = this.length; + } + + if (start < 0 || end > target.length || thisStart < 0 || thisEnd > this.length) { + throw new RangeError('out of range index') + } + + if (thisStart >= thisEnd && start >= end) { + return 0 + } + if (thisStart >= thisEnd) { + return -1 + } + if (start >= end) { + return 1 + } + + start >>>= 0; + end >>>= 0; + thisStart >>>= 0; + thisEnd >>>= 0; + + if (this === target) return 0 + + var x = thisEnd - thisStart; + var y = end - start; + var len = Math.min(x, y); + + var thisCopy = this.slice(thisStart, thisEnd); + var targetCopy = target.slice(start, end); + + for (var i = 0; i < len; ++i) { + if (thisCopy[i] !== targetCopy[i]) { + x = thisCopy[i]; + y = targetCopy[i]; + break + } + } + + if (x < y) return -1 + if (y < x) return 1 + return 0 + }; + + // Finds either the first index of `val` in `buffer` at offset >= `byteOffset`, + // OR the last index of `val` in `buffer` at offset <= `byteOffset`. + // + // Arguments: + // - buffer - a Buffer to search + // - val - a string, Buffer, or number + // - byteOffset - an index into `buffer`; will be clamped to an int32 + // - encoding - an optional encoding, relevant is val is a string + // - dir - true for indexOf, false for lastIndexOf + function bidirectionalIndexOf (buffer, val, byteOffset, encoding, dir) { + // Empty buffer means no match + if (buffer.length === 0) return -1 + + // Normalize byteOffset + if (typeof byteOffset === 'string') { + encoding = byteOffset; + byteOffset = 0; + } else if (byteOffset > 0x7fffffff) { + byteOffset = 0x7fffffff; + } else if (byteOffset < -0x80000000) { + byteOffset = -0x80000000; + } + byteOffset = +byteOffset; // Coerce to Number. + if (isNaN(byteOffset)) { + // byteOffset: it it's undefined, null, NaN, "foo", etc, search whole buffer + byteOffset = dir ? 0 : (buffer.length - 1); + } + + // Normalize byteOffset: negative offsets start from the end of the buffer + if (byteOffset < 0) byteOffset = buffer.length + byteOffset; + if (byteOffset >= buffer.length) { + if (dir) return -1 + else byteOffset = buffer.length - 1; + } else if (byteOffset < 0) { + if (dir) byteOffset = 0; + else return -1 + } + + // Normalize val + if (typeof val === 'string') { + val = Buffer.from(val, encoding); + } + + // Finally, search either indexOf (if dir is true) or lastIndexOf + if (internalIsBuffer(val)) { + // Special case: looking for empty string/buffer always fails + if (val.length === 0) { + return -1 + } + return arrayIndexOf(buffer, val, byteOffset, encoding, dir) + } else if (typeof val === 'number') { + val = val & 0xFF; // Search for a byte value [0-255] + if (Buffer.TYPED_ARRAY_SUPPORT && + typeof Uint8Array.prototype.indexOf === 'function') { + if (dir) { + return Uint8Array.prototype.indexOf.call(buffer, val, byteOffset) + } else { + return Uint8Array.prototype.lastIndexOf.call(buffer, val, byteOffset) + } + } + return arrayIndexOf(buffer, [ val ], byteOffset, encoding, dir) + } + + throw new TypeError('val must be string, number or Buffer') + } + + function arrayIndexOf (arr, val, byteOffset, encoding, dir) { + var indexSize = 1; + var arrLength = arr.length; + var valLength = val.length; + + if (encoding !== undefined) { + encoding = String(encoding).toLowerCase(); + if (encoding === 'ucs2' || encoding === 'ucs-2' || + encoding === 'utf16le' || encoding === 'utf-16le') { + if (arr.length < 2 || val.length < 2) { + return -1 + } + indexSize = 2; + arrLength /= 2; + valLength /= 2; + byteOffset /= 2; + } + } + + function read (buf, i) { + if (indexSize === 1) { + return buf[i] + } else { + return buf.readUInt16BE(i * indexSize) + } + } + + var i; + if (dir) { + var foundIndex = -1; + for (i = byteOffset; i < arrLength; i++) { + if (read(arr, i) === read(val, foundIndex === -1 ? 0 : i - foundIndex)) { + if (foundIndex === -1) foundIndex = i; + if (i - foundIndex + 1 === valLength) return foundIndex * indexSize + } else { + if (foundIndex !== -1) i -= i - foundIndex; + foundIndex = -1; + } + } + } else { + if (byteOffset + valLength > arrLength) byteOffset = arrLength - valLength; + for (i = byteOffset; i >= 0; i--) { + var found = true; + for (var j = 0; j < valLength; j++) { + if (read(arr, i + j) !== read(val, j)) { + found = false; + break + } + } + if (found) return i + } + } + + return -1 + } + + Buffer.prototype.includes = function includes (val, byteOffset, encoding) { + return this.indexOf(val, byteOffset, encoding) !== -1 + }; + + Buffer.prototype.indexOf = function indexOf (val, byteOffset, encoding) { + return bidirectionalIndexOf(this, val, byteOffset, encoding, true) + }; + + Buffer.prototype.lastIndexOf = function lastIndexOf (val, byteOffset, encoding) { + return bidirectionalIndexOf(this, val, byteOffset, encoding, false) + }; + + function hexWrite (buf, string, offset, length) { + offset = Number(offset) || 0; + var remaining = buf.length - offset; + if (!length) { + length = remaining; + } else { + length = Number(length); + if (length > remaining) { + length = remaining; + } + } + + // must be an even number of digits + var strLen = string.length; + if (strLen % 2 !== 0) throw new TypeError('Invalid hex string') + + if (length > strLen / 2) { + length = strLen / 2; + } + for (var i = 0; i < length; ++i) { + var parsed = parseInt(string.substr(i * 2, 2), 16); + if (isNaN(parsed)) return i + buf[offset + i] = parsed; + } + return i + } + + function utf8Write (buf, string, offset, length) { + return blitBuffer(utf8ToBytes(string, buf.length - offset), buf, offset, length) + } + + function asciiWrite (buf, string, offset, length) { + return blitBuffer(asciiToBytes(string), buf, offset, length) + } + + function latin1Write (buf, string, offset, length) { + return asciiWrite(buf, string, offset, length) + } + + function base64Write (buf, string, offset, length) { + return blitBuffer(base64ToBytes(string), buf, offset, length) + } + + function ucs2Write (buf, string, offset, length) { + return blitBuffer(utf16leToBytes(string, buf.length - offset), buf, offset, length) + } + + Buffer.prototype.write = function write (string, offset, length, encoding) { + // Buffer#write(string) + if (offset === undefined) { + encoding = 'utf8'; + length = this.length; + offset = 0; + // Buffer#write(string, encoding) + } else if (length === undefined && typeof offset === 'string') { + encoding = offset; + length = this.length; + offset = 0; + // Buffer#write(string, offset[, length][, encoding]) + } else if (isFinite(offset)) { + offset = offset | 0; + if (isFinite(length)) { + length = length | 0; + if (encoding === undefined) encoding = 'utf8'; + } else { + encoding = length; + length = undefined; + } + // legacy write(string, encoding, offset, length) - remove in v0.13 + } else { + throw new Error( + 'Buffer.write(string, encoding, offset[, length]) is no longer supported' + ) + } + + var remaining = this.length - offset; + if (length === undefined || length > remaining) length = remaining; + + if ((string.length > 0 && (length < 0 || offset < 0)) || offset > this.length) { + throw new RangeError('Attempt to write outside buffer bounds') + } + + if (!encoding) encoding = 'utf8'; + + var loweredCase = false; + for (;;) { + switch (encoding) { + case 'hex': + return hexWrite(this, string, offset, length) + + case 'utf8': + case 'utf-8': + return utf8Write(this, string, offset, length) + + case 'ascii': + return asciiWrite(this, string, offset, length) + + case 'latin1': + case 'binary': + return latin1Write(this, string, offset, length) + + case 'base64': + // Warning: maxLength not taken into account in base64Write + return base64Write(this, string, offset, length) + + case 'ucs2': + case 'ucs-2': + case 'utf16le': + case 'utf-16le': + return ucs2Write(this, string, offset, length) + + default: + if (loweredCase) throw new TypeError('Unknown encoding: ' + encoding) + encoding = ('' + encoding).toLowerCase(); + loweredCase = true; + } + } + }; + + Buffer.prototype.toJSON = function toJSON () { + return { + type: 'Buffer', + data: Array.prototype.slice.call(this._arr || this, 0) + } + }; + + function base64Slice (buf, start, end) { + if (start === 0 && end === buf.length) { + return fromByteArray(buf) + } else { + return fromByteArray(buf.slice(start, end)) + } + } + + function utf8Slice (buf, start, end) { + end = Math.min(buf.length, end); + var res = []; + + var i = start; + while (i < end) { + var firstByte = buf[i]; + var codePoint = null; + var bytesPerSequence = (firstByte > 0xEF) ? 4 + : (firstByte > 0xDF) ? 3 + : (firstByte > 0xBF) ? 2 + : 1; + + if (i + bytesPerSequence <= end) { + var secondByte, thirdByte, fourthByte, tempCodePoint; + + switch (bytesPerSequence) { + case 1: + if (firstByte < 0x80) { + codePoint = firstByte; + } + break + case 2: + secondByte = buf[i + 1]; + if ((secondByte & 0xC0) === 0x80) { + tempCodePoint = (firstByte & 0x1F) << 0x6 | (secondByte & 0x3F); + if (tempCodePoint > 0x7F) { + codePoint = tempCodePoint; + } + } + break + case 3: + secondByte = buf[i + 1]; + thirdByte = buf[i + 2]; + if ((secondByte & 0xC0) === 0x80 && (thirdByte & 0xC0) === 0x80) { + tempCodePoint = (firstByte & 0xF) << 0xC | (secondByte & 0x3F) << 0x6 | (thirdByte & 0x3F); + if (tempCodePoint > 0x7FF && (tempCodePoint < 0xD800 || tempCodePoint > 0xDFFF)) { + codePoint = tempCodePoint; + } + } + break + case 4: + secondByte = buf[i + 1]; + thirdByte = buf[i + 2]; + fourthByte = buf[i + 3]; + if ((secondByte & 0xC0) === 0x80 && (thirdByte & 0xC0) === 0x80 && (fourthByte & 0xC0) === 0x80) { + tempCodePoint = (firstByte & 0xF) << 0x12 | (secondByte & 0x3F) << 0xC | (thirdByte & 0x3F) << 0x6 | (fourthByte & 0x3F); + if (tempCodePoint > 0xFFFF && tempCodePoint < 0x110000) { + codePoint = tempCodePoint; + } + } + } + } + + if (codePoint === null) { + // we did not generate a valid codePoint so insert a + // replacement char (U+FFFD) and advance only 1 byte + codePoint = 0xFFFD; + bytesPerSequence = 1; + } else if (codePoint > 0xFFFF) { + // encode to utf16 (surrogate pair dance) + codePoint -= 0x10000; + res.push(codePoint >>> 10 & 0x3FF | 0xD800); + codePoint = 0xDC00 | codePoint & 0x3FF; + } + + res.push(codePoint); + i += bytesPerSequence; + } + + return decodeCodePointsArray(res) + } + + // Based on http://stackoverflow.com/a/22747272/680742, the browser with + // the lowest limit is Chrome, with 0x10000 args. + // We go 1 magnitude less, for safety + var MAX_ARGUMENTS_LENGTH = 0x1000; + + function decodeCodePointsArray (codePoints) { + var len = codePoints.length; + if (len <= MAX_ARGUMENTS_LENGTH) { + return String.fromCharCode.apply(String, codePoints) // avoid extra slice() + } + + // Decode in chunks to avoid "call stack size exceeded". + var res = ''; + var i = 0; + while (i < len) { + res += String.fromCharCode.apply( + String, + codePoints.slice(i, i += MAX_ARGUMENTS_LENGTH) + ); + } + return res + } + + function asciiSlice (buf, start, end) { + var ret = ''; + end = Math.min(buf.length, end); + + for (var i = start; i < end; ++i) { + ret += String.fromCharCode(buf[i] & 0x7F); + } + return ret + } + + function latin1Slice (buf, start, end) { + var ret = ''; + end = Math.min(buf.length, end); + + for (var i = start; i < end; ++i) { + ret += String.fromCharCode(buf[i]); + } + return ret + } + + function hexSlice (buf, start, end) { + var len = buf.length; + + if (!start || start < 0) start = 0; + if (!end || end < 0 || end > len) end = len; + + var out = ''; + for (var i = start; i < end; ++i) { + out += toHex(buf[i]); + } + return out + } + + function utf16leSlice (buf, start, end) { + var bytes = buf.slice(start, end); + var res = ''; + for (var i = 0; i < bytes.length; i += 2) { + res += String.fromCharCode(bytes[i] + bytes[i + 1] * 256); + } + return res + } + + Buffer.prototype.slice = function slice (start, end) { + var len = this.length; + start = ~~start; + end = end === undefined ? len : ~~end; + + if (start < 0) { + start += len; + if (start < 0) start = 0; + } else if (start > len) { + start = len; + } + + if (end < 0) { + end += len; + if (end < 0) end = 0; + } else if (end > len) { + end = len; + } + + if (end < start) end = start; + + var newBuf; + if (Buffer.TYPED_ARRAY_SUPPORT) { + newBuf = this.subarray(start, end); + newBuf.__proto__ = Buffer.prototype; + } else { + var sliceLen = end - start; + newBuf = new Buffer(sliceLen, undefined); + for (var i = 0; i < sliceLen; ++i) { + newBuf[i] = this[i + start]; + } + } + + return newBuf + }; + + /* + * Need to make sure that buffer isn't trying to write out of bounds. + */ + function checkOffset (offset, ext, length) { + if ((offset % 1) !== 0 || offset < 0) throw new RangeError('offset is not uint') + if (offset + ext > length) throw new RangeError('Trying to access beyond buffer length') + } + + Buffer.prototype.readUIntLE = function readUIntLE (offset, byteLength, noAssert) { + offset = offset | 0; + byteLength = byteLength | 0; + if (!noAssert) checkOffset(offset, byteLength, this.length); + + var val = this[offset]; + var mul = 1; + var i = 0; + while (++i < byteLength && (mul *= 0x100)) { + val += this[offset + i] * mul; + } + + return val + }; + + Buffer.prototype.readUIntBE = function readUIntBE (offset, byteLength, noAssert) { + offset = offset | 0; + byteLength = byteLength | 0; + if (!noAssert) { + checkOffset(offset, byteLength, this.length); + } + + var val = this[offset + --byteLength]; + var mul = 1; + while (byteLength > 0 && (mul *= 0x100)) { + val += this[offset + --byteLength] * mul; + } + + return val + }; + + Buffer.prototype.readUInt8 = function readUInt8 (offset, noAssert) { + if (!noAssert) checkOffset(offset, 1, this.length); + return this[offset] + }; + + Buffer.prototype.readUInt16LE = function readUInt16LE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 2, this.length); + return this[offset] | (this[offset + 1] << 8) + }; + + Buffer.prototype.readUInt16BE = function readUInt16BE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 2, this.length); + return (this[offset] << 8) | this[offset + 1] + }; + + Buffer.prototype.readUInt32LE = function readUInt32LE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 4, this.length); + + return ((this[offset]) | + (this[offset + 1] << 8) | + (this[offset + 2] << 16)) + + (this[offset + 3] * 0x1000000) + }; + + Buffer.prototype.readUInt32BE = function readUInt32BE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 4, this.length); + + return (this[offset] * 0x1000000) + + ((this[offset + 1] << 16) | + (this[offset + 2] << 8) | + this[offset + 3]) + }; + + Buffer.prototype.readIntLE = function readIntLE (offset, byteLength, noAssert) { + offset = offset | 0; + byteLength = byteLength | 0; + if (!noAssert) checkOffset(offset, byteLength, this.length); + + var val = this[offset]; + var mul = 1; + var i = 0; + while (++i < byteLength && (mul *= 0x100)) { + val += this[offset + i] * mul; + } + mul *= 0x80; + + if (val >= mul) val -= Math.pow(2, 8 * byteLength); + + return val + }; + + Buffer.prototype.readIntBE = function readIntBE (offset, byteLength, noAssert) { + offset = offset | 0; + byteLength = byteLength | 0; + if (!noAssert) checkOffset(offset, byteLength, this.length); + + var i = byteLength; + var mul = 1; + var val = this[offset + --i]; + while (i > 0 && (mul *= 0x100)) { + val += this[offset + --i] * mul; + } + mul *= 0x80; + + if (val >= mul) val -= Math.pow(2, 8 * byteLength); + + return val + }; + + Buffer.prototype.readInt8 = function readInt8 (offset, noAssert) { + if (!noAssert) checkOffset(offset, 1, this.length); + if (!(this[offset] & 0x80)) return (this[offset]) + return ((0xff - this[offset] + 1) * -1) + }; + + Buffer.prototype.readInt16LE = function readInt16LE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 2, this.length); + var val = this[offset] | (this[offset + 1] << 8); + return (val & 0x8000) ? val | 0xFFFF0000 : val + }; + + Buffer.prototype.readInt16BE = function readInt16BE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 2, this.length); + var val = this[offset + 1] | (this[offset] << 8); + return (val & 0x8000) ? val | 0xFFFF0000 : val + }; + + Buffer.prototype.readInt32LE = function readInt32LE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 4, this.length); + + return (this[offset]) | + (this[offset + 1] << 8) | + (this[offset + 2] << 16) | + (this[offset + 3] << 24) + }; + + Buffer.prototype.readInt32BE = function readInt32BE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 4, this.length); + + return (this[offset] << 24) | + (this[offset + 1] << 16) | + (this[offset + 2] << 8) | + (this[offset + 3]) + }; + + Buffer.prototype.readFloatLE = function readFloatLE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 4, this.length); + return read(this, offset, true, 23, 4) + }; + + Buffer.prototype.readFloatBE = function readFloatBE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 4, this.length); + return read(this, offset, false, 23, 4) + }; + + Buffer.prototype.readDoubleLE = function readDoubleLE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 8, this.length); + return read(this, offset, true, 52, 8) + }; + + Buffer.prototype.readDoubleBE = function readDoubleBE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 8, this.length); + return read(this, offset, false, 52, 8) + }; + + function checkInt (buf, value, offset, ext, max, min) { + if (!internalIsBuffer(buf)) throw new TypeError('"buffer" argument must be a Buffer instance') + if (value > max || value < min) throw new RangeError('"value" argument is out of bounds') + if (offset + ext > buf.length) throw new RangeError('Index out of range') + } + + Buffer.prototype.writeUIntLE = function writeUIntLE (value, offset, byteLength, noAssert) { + value = +value; + offset = offset | 0; + byteLength = byteLength | 0; + if (!noAssert) { + var maxBytes = Math.pow(2, 8 * byteLength) - 1; + checkInt(this, value, offset, byteLength, maxBytes, 0); + } + + var mul = 1; + var i = 0; + this[offset] = value & 0xFF; + while (++i < byteLength && (mul *= 0x100)) { + this[offset + i] = (value / mul) & 0xFF; + } + + return offset + byteLength + }; + + Buffer.prototype.writeUIntBE = function writeUIntBE (value, offset, byteLength, noAssert) { + value = +value; + offset = offset | 0; + byteLength = byteLength | 0; + if (!noAssert) { + var maxBytes = Math.pow(2, 8 * byteLength) - 1; + checkInt(this, value, offset, byteLength, maxBytes, 0); + } + + var i = byteLength - 1; + var mul = 1; + this[offset + i] = value & 0xFF; + while (--i >= 0 && (mul *= 0x100)) { + this[offset + i] = (value / mul) & 0xFF; + } + + return offset + byteLength + }; + + Buffer.prototype.writeUInt8 = function writeUInt8 (value, offset, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) checkInt(this, value, offset, 1, 0xff, 0); + if (!Buffer.TYPED_ARRAY_SUPPORT) value = Math.floor(value); + this[offset] = (value & 0xff); + return offset + 1 + }; + + function objectWriteUInt16 (buf, value, offset, littleEndian) { + if (value < 0) value = 0xffff + value + 1; + for (var i = 0, j = Math.min(buf.length - offset, 2); i < j; ++i) { + buf[offset + i] = (value & (0xff << (8 * (littleEndian ? i : 1 - i)))) >>> + (littleEndian ? i : 1 - i) * 8; + } + } + + Buffer.prototype.writeUInt16LE = function writeUInt16LE (value, offset, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) checkInt(this, value, offset, 2, 0xffff, 0); + if (Buffer.TYPED_ARRAY_SUPPORT) { + this[offset] = (value & 0xff); + this[offset + 1] = (value >>> 8); + } else { + objectWriteUInt16(this, value, offset, true); + } + return offset + 2 + }; + + Buffer.prototype.writeUInt16BE = function writeUInt16BE (value, offset, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) checkInt(this, value, offset, 2, 0xffff, 0); + if (Buffer.TYPED_ARRAY_SUPPORT) { + this[offset] = (value >>> 8); + this[offset + 1] = (value & 0xff); + } else { + objectWriteUInt16(this, value, offset, false); + } + return offset + 2 + }; + + function objectWriteUInt32 (buf, value, offset, littleEndian) { + if (value < 0) value = 0xffffffff + value + 1; + for (var i = 0, j = Math.min(buf.length - offset, 4); i < j; ++i) { + buf[offset + i] = (value >>> (littleEndian ? i : 3 - i) * 8) & 0xff; + } + } + + Buffer.prototype.writeUInt32LE = function writeUInt32LE (value, offset, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) checkInt(this, value, offset, 4, 0xffffffff, 0); + if (Buffer.TYPED_ARRAY_SUPPORT) { + this[offset + 3] = (value >>> 24); + this[offset + 2] = (value >>> 16); + this[offset + 1] = (value >>> 8); + this[offset] = (value & 0xff); + } else { + objectWriteUInt32(this, value, offset, true); + } + return offset + 4 + }; + + Buffer.prototype.writeUInt32BE = function writeUInt32BE (value, offset, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) checkInt(this, value, offset, 4, 0xffffffff, 0); + if (Buffer.TYPED_ARRAY_SUPPORT) { + this[offset] = (value >>> 24); + this[offset + 1] = (value >>> 16); + this[offset + 2] = (value >>> 8); + this[offset + 3] = (value & 0xff); + } else { + objectWriteUInt32(this, value, offset, false); + } + return offset + 4 + }; + + Buffer.prototype.writeIntLE = function writeIntLE (value, offset, byteLength, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) { + var limit = Math.pow(2, 8 * byteLength - 1); + + checkInt(this, value, offset, byteLength, limit - 1, -limit); + } + + var i = 0; + var mul = 1; + var sub = 0; + this[offset] = value & 0xFF; + while (++i < byteLength && (mul *= 0x100)) { + if (value < 0 && sub === 0 && this[offset + i - 1] !== 0) { + sub = 1; + } + this[offset + i] = ((value / mul) >> 0) - sub & 0xFF; + } + + return offset + byteLength + }; + + Buffer.prototype.writeIntBE = function writeIntBE (value, offset, byteLength, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) { + var limit = Math.pow(2, 8 * byteLength - 1); + + checkInt(this, value, offset, byteLength, limit - 1, -limit); + } + + var i = byteLength - 1; + var mul = 1; + var sub = 0; + this[offset + i] = value & 0xFF; + while (--i >= 0 && (mul *= 0x100)) { + if (value < 0 && sub === 0 && this[offset + i + 1] !== 0) { + sub = 1; + } + this[offset + i] = ((value / mul) >> 0) - sub & 0xFF; + } + + return offset + byteLength + }; + + Buffer.prototype.writeInt8 = function writeInt8 (value, offset, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) checkInt(this, value, offset, 1, 0x7f, -0x80); + if (!Buffer.TYPED_ARRAY_SUPPORT) value = Math.floor(value); + if (value < 0) value = 0xff + value + 1; + this[offset] = (value & 0xff); + return offset + 1 + }; + + Buffer.prototype.writeInt16LE = function writeInt16LE (value, offset, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) checkInt(this, value, offset, 2, 0x7fff, -0x8000); + if (Buffer.TYPED_ARRAY_SUPPORT) { + this[offset] = (value & 0xff); + this[offset + 1] = (value >>> 8); + } else { + objectWriteUInt16(this, value, offset, true); + } + return offset + 2 + }; + + Buffer.prototype.writeInt16BE = function writeInt16BE (value, offset, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) checkInt(this, value, offset, 2, 0x7fff, -0x8000); + if (Buffer.TYPED_ARRAY_SUPPORT) { + this[offset] = (value >>> 8); + this[offset + 1] = (value & 0xff); + } else { + objectWriteUInt16(this, value, offset, false); + } + return offset + 2 + }; + + Buffer.prototype.writeInt32LE = function writeInt32LE (value, offset, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) checkInt(this, value, offset, 4, 0x7fffffff, -0x80000000); + if (Buffer.TYPED_ARRAY_SUPPORT) { + this[offset] = (value & 0xff); + this[offset + 1] = (value >>> 8); + this[offset + 2] = (value >>> 16); + this[offset + 3] = (value >>> 24); + } else { + objectWriteUInt32(this, value, offset, true); + } + return offset + 4 + }; + + Buffer.prototype.writeInt32BE = function writeInt32BE (value, offset, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) checkInt(this, value, offset, 4, 0x7fffffff, -0x80000000); + if (value < 0) value = 0xffffffff + value + 1; + if (Buffer.TYPED_ARRAY_SUPPORT) { + this[offset] = (value >>> 24); + this[offset + 1] = (value >>> 16); + this[offset + 2] = (value >>> 8); + this[offset + 3] = (value & 0xff); + } else { + objectWriteUInt32(this, value, offset, false); + } + return offset + 4 + }; + + function checkIEEE754 (buf, value, offset, ext, max, min) { + if (offset + ext > buf.length) throw new RangeError('Index out of range') + if (offset < 0) throw new RangeError('Index out of range') + } + + function writeFloat (buf, value, offset, littleEndian, noAssert) { + if (!noAssert) { + checkIEEE754(buf, value, offset, 4); + } + write(buf, value, offset, littleEndian, 23, 4); + return offset + 4 + } + + Buffer.prototype.writeFloatLE = function writeFloatLE (value, offset, noAssert) { + return writeFloat(this, value, offset, true, noAssert) + }; + + Buffer.prototype.writeFloatBE = function writeFloatBE (value, offset, noAssert) { + return writeFloat(this, value, offset, false, noAssert) + }; + + function writeDouble (buf, value, offset, littleEndian, noAssert) { + if (!noAssert) { + checkIEEE754(buf, value, offset, 8); + } + write(buf, value, offset, littleEndian, 52, 8); + return offset + 8 + } + + Buffer.prototype.writeDoubleLE = function writeDoubleLE (value, offset, noAssert) { + return writeDouble(this, value, offset, true, noAssert) + }; + + Buffer.prototype.writeDoubleBE = function writeDoubleBE (value, offset, noAssert) { + return writeDouble(this, value, offset, false, noAssert) + }; + + // copy(targetBuffer, targetStart=0, sourceStart=0, sourceEnd=buffer.length) + Buffer.prototype.copy = function copy (target, targetStart, start, end) { + if (!start) start = 0; + if (!end && end !== 0) end = this.length; + if (targetStart >= target.length) targetStart = target.length; + if (!targetStart) targetStart = 0; + if (end > 0 && end < start) end = start; + + // Copy 0 bytes; we're done + if (end === start) return 0 + if (target.length === 0 || this.length === 0) return 0 + + // Fatal error conditions + if (targetStart < 0) { + throw new RangeError('targetStart out of bounds') + } + if (start < 0 || start >= this.length) throw new RangeError('sourceStart out of bounds') + if (end < 0) throw new RangeError('sourceEnd out of bounds') + + // Are we oob? + if (end > this.length) end = this.length; + if (target.length - targetStart < end - start) { + end = target.length - targetStart + start; + } + + var len = end - start; + var i; + + if (this === target && start < targetStart && targetStart < end) { + // descending copy from end + for (i = len - 1; i >= 0; --i) { + target[i + targetStart] = this[i + start]; + } + } else if (len < 1000 || !Buffer.TYPED_ARRAY_SUPPORT) { + // ascending copy from start + for (i = 0; i < len; ++i) { + target[i + targetStart] = this[i + start]; + } + } else { + Uint8Array.prototype.set.call( + target, + this.subarray(start, start + len), + targetStart + ); + } + + return len + }; + + // Usage: + // buffer.fill(number[, offset[, end]]) + // buffer.fill(buffer[, offset[, end]]) + // buffer.fill(string[, offset[, end]][, encoding]) + Buffer.prototype.fill = function fill (val, start, end, encoding) { + // Handle string cases: + if (typeof val === 'string') { + if (typeof start === 'string') { + encoding = start; + start = 0; + end = this.length; + } else if (typeof end === 'string') { + encoding = end; + end = this.length; + } + if (val.length === 1) { + var code = val.charCodeAt(0); + if (code < 256) { + val = code; + } + } + if (encoding !== undefined && typeof encoding !== 'string') { + throw new TypeError('encoding must be a string') + } + if (typeof encoding === 'string' && !Buffer.isEncoding(encoding)) { + throw new TypeError('Unknown encoding: ' + encoding) + } + } else if (typeof val === 'number') { + val = val & 255; + } + + // Invalid ranges are not set to a default, so can range check early. + if (start < 0 || this.length < start || this.length < end) { + throw new RangeError('Out of range index') + } + + if (end <= start) { + return this + } + + start = start >>> 0; + end = end === undefined ? this.length : end >>> 0; + + if (!val) val = 0; + + var i; + if (typeof val === 'number') { + for (i = start; i < end; ++i) { + this[i] = val; + } + } else { + var bytes = internalIsBuffer(val) + ? val + : utf8ToBytes(new Buffer(val, encoding).toString()); + var len = bytes.length; + for (i = 0; i < end - start; ++i) { + this[i + start] = bytes[i % len]; + } + } + + return this + }; + + // HELPER FUNCTIONS + // ================ + + var INVALID_BASE64_RE = /[^+\/0-9A-Za-z-_]/g; + + function base64clean (str) { + // Node strips out invalid characters like \n and \t from the string, base64-js does not + str = stringtrim(str).replace(INVALID_BASE64_RE, ''); + // Node converts strings with length < 2 to '' + if (str.length < 2) return '' + // Node allows for non-padded base64 strings (missing trailing ===), base64-js does not + while (str.length % 4 !== 0) { + str = str + '='; + } + return str + } + + function stringtrim (str) { + if (str.trim) return str.trim() + return str.replace(/^\s+|\s+$/g, '') + } + + function toHex (n) { + if (n < 16) return '0' + n.toString(16) + return n.toString(16) + } + + function utf8ToBytes (string, units) { + units = units || Infinity; + var codePoint; + var length = string.length; + var leadSurrogate = null; + var bytes = []; + + for (var i = 0; i < length; ++i) { + codePoint = string.charCodeAt(i); + + // is surrogate component + if (codePoint > 0xD7FF && codePoint < 0xE000) { + // last char was a lead + if (!leadSurrogate) { + // no lead yet + if (codePoint > 0xDBFF) { + // unexpected trail + if ((units -= 3) > -1) bytes.push(0xEF, 0xBF, 0xBD); + continue + } else if (i + 1 === length) { + // unpaired lead + if ((units -= 3) > -1) bytes.push(0xEF, 0xBF, 0xBD); + continue + } + + // valid lead + leadSurrogate = codePoint; + + continue + } + + // 2 leads in a row + if (codePoint < 0xDC00) { + if ((units -= 3) > -1) bytes.push(0xEF, 0xBF, 0xBD); + leadSurrogate = codePoint; + continue + } + + // valid surrogate pair + codePoint = (leadSurrogate - 0xD800 << 10 | codePoint - 0xDC00) + 0x10000; + } else if (leadSurrogate) { + // valid bmp char, but last char was a lead + if ((units -= 3) > -1) bytes.push(0xEF, 0xBF, 0xBD); + } + + leadSurrogate = null; + + // encode utf8 + if (codePoint < 0x80) { + if ((units -= 1) < 0) break + bytes.push(codePoint); + } else if (codePoint < 0x800) { + if ((units -= 2) < 0) break + bytes.push( + codePoint >> 0x6 | 0xC0, + codePoint & 0x3F | 0x80 + ); + } else if (codePoint < 0x10000) { + if ((units -= 3) < 0) break + bytes.push( + codePoint >> 0xC | 0xE0, + codePoint >> 0x6 & 0x3F | 0x80, + codePoint & 0x3F | 0x80 + ); + } else if (codePoint < 0x110000) { + if ((units -= 4) < 0) break + bytes.push( + codePoint >> 0x12 | 0xF0, + codePoint >> 0xC & 0x3F | 0x80, + codePoint >> 0x6 & 0x3F | 0x80, + codePoint & 0x3F | 0x80 + ); + } else { + throw new Error('Invalid code point') + } + } + + return bytes + } + + function asciiToBytes (str) { + var byteArray = []; + for (var i = 0; i < str.length; ++i) { + // Node's code seems to be doing this and not & 0x7F.. + byteArray.push(str.charCodeAt(i) & 0xFF); + } + return byteArray + } + + function utf16leToBytes (str, units) { + var c, hi, lo; + var byteArray = []; + for (var i = 0; i < str.length; ++i) { + if ((units -= 2) < 0) break + + c = str.charCodeAt(i); + hi = c >> 8; + lo = c % 256; + byteArray.push(lo); + byteArray.push(hi); + } + + return byteArray + } + + + function base64ToBytes (str) { + return toByteArray(base64clean(str)) + } + + function blitBuffer (src, dst, offset, length) { + for (var i = 0; i < length; ++i) { + if ((i + offset >= dst.length) || (i >= src.length)) break + dst[i + offset] = src[i]; + } + return i + } + + function isnan (val) { + return val !== val // eslint-disable-line no-self-compare + } + + + // the following is from is-buffer, also by Feross Aboukhadijeh and with same lisence + // The _isBuffer check is for Safari 5-7 support, because it's missing + // Object.prototype.constructor. Remove this eventually + function isBuffer(obj) { + return obj != null && (!!obj._isBuffer || isFastBuffer(obj) || isSlowBuffer(obj)) + } + + function isFastBuffer (obj) { + return !!obj.constructor && typeof obj.constructor.isBuffer === 'function' && obj.constructor.isBuffer(obj) + } + + // For Node v0.10 support. Remove this eventually. + function isSlowBuffer (obj) { + return typeof obj.readFloatLE === 'function' && typeof obj.slice === 'function' && isFastBuffer(obj.slice(0, 0)) + } + + // shim for using process in browser + // based off https://github.com/defunctzombie/node-process/blob/master/browser.js + + function defaultSetTimout() { + throw new Error('setTimeout has not been defined'); + } + function defaultClearTimeout () { + throw new Error('clearTimeout has not been defined'); + } + var cachedSetTimeout = defaultSetTimout; + var cachedClearTimeout = defaultClearTimeout; + if (typeof global$1.setTimeout === 'function') { + cachedSetTimeout = setTimeout; + } + if (typeof global$1.clearTimeout === 'function') { + cachedClearTimeout = clearTimeout; + } + + function runTimeout(fun) { + if (cachedSetTimeout === setTimeout) { + //normal enviroments in sane situations + return setTimeout(fun, 0); + } + // if setTimeout wasn't available but was latter defined + if ((cachedSetTimeout === defaultSetTimout || !cachedSetTimeout) && setTimeout) { + cachedSetTimeout = setTimeout; + return setTimeout(fun, 0); + } + try { + // when when somebody has screwed with setTimeout but no I.E. maddness + return cachedSetTimeout(fun, 0); + } catch(e){ + try { + // When we are in I.E. but the script has been evaled so I.E. doesn't trust the global object when called normally + return cachedSetTimeout.call(null, fun, 0); + } catch(e){ + // same as above but when it's a version of I.E. that must have the global object for 'this', hopfully our context correct otherwise it will throw a global error + return cachedSetTimeout.call(this, fun, 0); + } + } + + + } + function runClearTimeout(marker) { + if (cachedClearTimeout === clearTimeout) { + //normal enviroments in sane situations + return clearTimeout(marker); + } + // if clearTimeout wasn't available but was latter defined + if ((cachedClearTimeout === defaultClearTimeout || !cachedClearTimeout) && clearTimeout) { + cachedClearTimeout = clearTimeout; + return clearTimeout(marker); + } + try { + // when when somebody has screwed with setTimeout but no I.E. maddness + return cachedClearTimeout(marker); + } catch (e){ + try { + // When we are in I.E. but the script has been evaled so I.E. doesn't trust the global object when called normally + return cachedClearTimeout.call(null, marker); + } catch (e){ + // same as above but when it's a version of I.E. that must have the global object for 'this', hopfully our context correct otherwise it will throw a global error. + // Some versions of I.E. have different rules for clearTimeout vs setTimeout + return cachedClearTimeout.call(this, marker); + } + } + + + + } + var queue = []; + var draining = false; + var currentQueue; + var queueIndex = -1; + + function cleanUpNextTick() { + if (!draining || !currentQueue) { + return; + } + draining = false; + if (currentQueue.length) { + queue = currentQueue.concat(queue); + } else { + queueIndex = -1; + } + if (queue.length) { + drainQueue(); + } + } + + function drainQueue() { + if (draining) { + return; + } + var timeout = runTimeout(cleanUpNextTick); + draining = true; + + var len = queue.length; + while(len) { + currentQueue = queue; + queue = []; + while (++queueIndex < len) { + if (currentQueue) { + currentQueue[queueIndex].run(); + } + } + queueIndex = -1; + len = queue.length; + } + currentQueue = null; + draining = false; + runClearTimeout(timeout); + } + function nextTick(fun) { + var args = new Array(arguments.length - 1); + if (arguments.length > 1) { + for (var i = 1; i < arguments.length; i++) { + args[i - 1] = arguments[i]; + } + } + queue.push(new Item(fun, args)); + if (queue.length === 1 && !draining) { + runTimeout(drainQueue); + } + } + // v8 likes predictible objects + function Item(fun, array) { + this.fun = fun; + this.array = array; + } + Item.prototype.run = function () { + this.fun.apply(null, this.array); + }; + + // from https://github.com/kumavis/browser-process-hrtime/blob/master/index.js + var performance = global$1.performance || {}; + performance.now || + performance.mozNow || + performance.msNow || + performance.oNow || + performance.webkitNow || + function(){ return (new Date()).getTime() }; + + var inherits; + if (typeof Object.create === 'function'){ + inherits = function inherits(ctor, superCtor) { + // implementation from standard node.js 'util' module + ctor.super_ = superCtor; + ctor.prototype = Object.create(superCtor.prototype, { + constructor: { + value: ctor, + enumerable: false, + writable: true, + configurable: true + } + }); + }; + } else { + inherits = function inherits(ctor, superCtor) { + ctor.super_ = superCtor; + var TempCtor = function () {}; + TempCtor.prototype = superCtor.prototype; + ctor.prototype = new TempCtor(); + ctor.prototype.constructor = ctor; + }; + } + var inherits$1 = inherits; + + var formatRegExp = /%[sdj%]/g; + function format(f) { + if (!isString(f)) { + var objects = []; + for (var i = 0; i < arguments.length; i++) { + objects.push(inspect(arguments[i])); + } + return objects.join(' '); + } + + var i = 1; + var args = arguments; + var len = args.length; + var str = String(f).replace(formatRegExp, function(x) { + if (x === '%%') return '%'; + if (i >= len) return x; + switch (x) { + case '%s': return String(args[i++]); + case '%d': return Number(args[i++]); + case '%j': + try { + return JSON.stringify(args[i++]); + } catch (_) { + return '[Circular]'; + } + default: + return x; + } + }); + for (var x = args[i]; i < len; x = args[++i]) { + if (isNull(x) || !isObject(x)) { + str += ' ' + x; + } else { + str += ' ' + inspect(x); + } + } + return str; + } + + // Mark that a method should not be used. + // Returns a modified function which warns once by default. + // If --no-deprecation is set, then it is a no-op. + function deprecate(fn, msg) { + // Allow for deprecating things in the process of starting up. + if (isUndefined(global$1.process)) { + return function() { + return deprecate(fn, msg).apply(this, arguments); + }; + } + + var warned = false; + function deprecated() { + if (!warned) { + { + console.error(msg); + } + warned = true; + } + return fn.apply(this, arguments); + } + + return deprecated; + } + + var debugs = {}; + var debugEnviron; + function debuglog(set) { + if (isUndefined(debugEnviron)) + debugEnviron = ''; + set = set.toUpperCase(); + if (!debugs[set]) { + if (new RegExp('\\b' + set + '\\b', 'i').test(debugEnviron)) { + var pid = 0; + debugs[set] = function() { + var msg = format.apply(null, arguments); + console.error('%s %d: %s', set, pid, msg); + }; + } else { + debugs[set] = function() {}; + } + } + return debugs[set]; + } + + /** + * Echos the value of a value. Trys to print the value out + * in the best way possible given the different types. + * + * @param {Object} obj The object to print out. + * @param {Object} opts Optional options object that alters the output. + */ + /* legacy: obj, showHidden, depth, colors*/ + function inspect(obj, opts) { + // default options + var ctx = { + seen: [], + stylize: stylizeNoColor + }; + // legacy... + if (arguments.length >= 3) ctx.depth = arguments[2]; + if (arguments.length >= 4) ctx.colors = arguments[3]; + if (isBoolean(opts)) { + // legacy... + ctx.showHidden = opts; + } else if (opts) { + // got an "options" object + _extend(ctx, opts); + } + // set default options + if (isUndefined(ctx.showHidden)) ctx.showHidden = false; + if (isUndefined(ctx.depth)) ctx.depth = 2; + if (isUndefined(ctx.colors)) ctx.colors = false; + if (isUndefined(ctx.customInspect)) ctx.customInspect = true; + if (ctx.colors) ctx.stylize = stylizeWithColor; + return formatValue(ctx, obj, ctx.depth); + } + + // http://en.wikipedia.org/wiki/ANSI_escape_code#graphics + inspect.colors = { + 'bold' : [1, 22], + 'italic' : [3, 23], + 'underline' : [4, 24], + 'inverse' : [7, 27], + 'white' : [37, 39], + 'grey' : [90, 39], + 'black' : [30, 39], + 'blue' : [34, 39], + 'cyan' : [36, 39], + 'green' : [32, 39], + 'magenta' : [35, 39], + 'red' : [31, 39], + 'yellow' : [33, 39] + }; + + // Don't use 'blue' not visible on cmd.exe + inspect.styles = { + 'special': 'cyan', + 'number': 'yellow', + 'boolean': 'yellow', + 'undefined': 'grey', + 'null': 'bold', + 'string': 'green', + 'date': 'magenta', + // "name": intentionally not styling + 'regexp': 'red' + }; + + + function stylizeWithColor(str, styleType) { + var style = inspect.styles[styleType]; + + if (style) { + return '\u001b[' + inspect.colors[style][0] + 'm' + str + + '\u001b[' + inspect.colors[style][1] + 'm'; + } else { + return str; + } + } + + + function stylizeNoColor(str, styleType) { + return str; + } + + + function arrayToHash(array) { + var hash = {}; + + array.forEach(function(val, idx) { + hash[val] = true; + }); + + return hash; + } + + + function formatValue(ctx, value, recurseTimes) { + // Provide a hook for user-specified inspect functions. + // Check that value is an object with an inspect function on it + if (ctx.customInspect && + value && + isFunction(value.inspect) && + // Filter out the util module, it's inspect function is special + value.inspect !== inspect && + // Also filter out any prototype objects using the circular check. + !(value.constructor && value.constructor.prototype === value)) { + var ret = value.inspect(recurseTimes, ctx); + if (!isString(ret)) { + ret = formatValue(ctx, ret, recurseTimes); + } + return ret; + } + + // Primitive types cannot have properties + var primitive = formatPrimitive(ctx, value); + if (primitive) { + return primitive; + } + + // Look up the keys of the object. + var keys = Object.keys(value); + var visibleKeys = arrayToHash(keys); + + if (ctx.showHidden) { + keys = Object.getOwnPropertyNames(value); + } + + // IE doesn't make error fields non-enumerable + // http://msdn.microsoft.com/en-us/library/ie/dww52sbt(v=vs.94).aspx + if (isError(value) + && (keys.indexOf('message') >= 0 || keys.indexOf('description') >= 0)) { + return formatError(value); + } + + // Some type of object without properties can be shortcutted. + if (keys.length === 0) { + if (isFunction(value)) { + var name = value.name ? ': ' + value.name : ''; + return ctx.stylize('[Function' + name + ']', 'special'); + } + if (isRegExp(value)) { + return ctx.stylize(RegExp.prototype.toString.call(value), 'regexp'); + } + if (isDate(value)) { + return ctx.stylize(Date.prototype.toString.call(value), 'date'); + } + if (isError(value)) { + return formatError(value); + } + } + + var base = '', array = false, braces = ['{', '}']; + + // Make Array say that they are Array + if (isArray(value)) { + array = true; + braces = ['[', ']']; + } + + // Make functions say that they are functions + if (isFunction(value)) { + var n = value.name ? ': ' + value.name : ''; + base = ' [Function' + n + ']'; + } + + // Make RegExps say that they are RegExps + if (isRegExp(value)) { + base = ' ' + RegExp.prototype.toString.call(value); + } + + // Make dates with properties first say the date + if (isDate(value)) { + base = ' ' + Date.prototype.toUTCString.call(value); + } + + // Make error with message first say the error + if (isError(value)) { + base = ' ' + formatError(value); + } + + if (keys.length === 0 && (!array || value.length == 0)) { + return braces[0] + base + braces[1]; + } + + if (recurseTimes < 0) { + if (isRegExp(value)) { + return ctx.stylize(RegExp.prototype.toString.call(value), 'regexp'); + } else { + return ctx.stylize('[Object]', 'special'); + } + } + + ctx.seen.push(value); + + var output; + if (array) { + output = formatArray(ctx, value, recurseTimes, visibleKeys, keys); + } else { + output = keys.map(function(key) { + return formatProperty(ctx, value, recurseTimes, visibleKeys, key, array); + }); + } + + ctx.seen.pop(); + + return reduceToSingleString(output, base, braces); + } + + + function formatPrimitive(ctx, value) { + if (isUndefined(value)) + return ctx.stylize('undefined', 'undefined'); + if (isString(value)) { + var simple = '\'' + JSON.stringify(value).replace(/^"|"$/g, '') + .replace(/'/g, "\\'") + .replace(/\\"/g, '"') + '\''; + return ctx.stylize(simple, 'string'); + } + if (isNumber(value)) + return ctx.stylize('' + value, 'number'); + if (isBoolean(value)) + return ctx.stylize('' + value, 'boolean'); + // For some reason typeof null is "object", so special case here. + if (isNull(value)) + return ctx.stylize('null', 'null'); + } + + + function formatError(value) { + return '[' + Error.prototype.toString.call(value) + ']'; + } + + + function formatArray(ctx, value, recurseTimes, visibleKeys, keys) { + var output = []; + for (var i = 0, l = value.length; i < l; ++i) { + if (hasOwnProperty(value, String(i))) { + output.push(formatProperty(ctx, value, recurseTimes, visibleKeys, + String(i), true)); + } else { + output.push(''); + } + } + keys.forEach(function(key) { + if (!key.match(/^\d+$/)) { + output.push(formatProperty(ctx, value, recurseTimes, visibleKeys, + key, true)); + } + }); + return output; + } + + + function formatProperty(ctx, value, recurseTimes, visibleKeys, key, array) { + var name, str, desc; + desc = Object.getOwnPropertyDescriptor(value, key) || { value: value[key] }; + if (desc.get) { + if (desc.set) { + str = ctx.stylize('[Getter/Setter]', 'special'); + } else { + str = ctx.stylize('[Getter]', 'special'); + } + } else { + if (desc.set) { + str = ctx.stylize('[Setter]', 'special'); + } + } + if (!hasOwnProperty(visibleKeys, key)) { + name = '[' + key + ']'; + } + if (!str) { + if (ctx.seen.indexOf(desc.value) < 0) { + if (isNull(recurseTimes)) { + str = formatValue(ctx, desc.value, null); + } else { + str = formatValue(ctx, desc.value, recurseTimes - 1); + } + if (str.indexOf('\n') > -1) { + if (array) { + str = str.split('\n').map(function(line) { + return ' ' + line; + }).join('\n').substr(2); + } else { + str = '\n' + str.split('\n').map(function(line) { + return ' ' + line; + }).join('\n'); + } + } + } else { + str = ctx.stylize('[Circular]', 'special'); + } + } + if (isUndefined(name)) { + if (array && key.match(/^\d+$/)) { + return str; + } + name = JSON.stringify('' + key); + if (name.match(/^"([a-zA-Z_][a-zA-Z_0-9]*)"$/)) { + name = name.substr(1, name.length - 2); + name = ctx.stylize(name, 'name'); + } else { + name = name.replace(/'/g, "\\'") + .replace(/\\"/g, '"') + .replace(/(^"|"$)/g, "'"); + name = ctx.stylize(name, 'string'); + } + } + + return name + ': ' + str; + } + + + function reduceToSingleString(output, base, braces) { + var length = output.reduce(function(prev, cur) { + if (cur.indexOf('\n') >= 0) ; + return prev + cur.replace(/\u001b\[\d\d?m/g, '').length + 1; + }, 0); + + if (length > 60) { + return braces[0] + + (base === '' ? '' : base + '\n ') + + ' ' + + output.join(',\n ') + + ' ' + + braces[1]; + } + + return braces[0] + base + ' ' + output.join(', ') + ' ' + braces[1]; + } + + + // NOTE: These type checking functions intentionally don't use `instanceof` + // because it is fragile and can be easily faked with `Object.create()`. + function isArray(ar) { + return Array.isArray(ar); + } + + function isBoolean(arg) { + return typeof arg === 'boolean'; + } + + function isNull(arg) { + return arg === null; + } + + function isNumber(arg) { + return typeof arg === 'number'; + } + + function isString(arg) { + return typeof arg === 'string'; + } + + function isUndefined(arg) { + return arg === void 0; + } + + function isRegExp(re) { + return isObject(re) && objectToString(re) === '[object RegExp]'; + } + + function isObject(arg) { + return typeof arg === 'object' && arg !== null; + } + + function isDate(d) { + return isObject(d) && objectToString(d) === '[object Date]'; + } + + function isError(e) { + return isObject(e) && + (objectToString(e) === '[object Error]' || e instanceof Error); + } + + function isFunction(arg) { + return typeof arg === 'function'; + } + + function objectToString(o) { + return Object.prototype.toString.call(o); + } + + function _extend(origin, add) { + // Don't do anything if add isn't an object + if (!add || !isObject(add)) return origin; + + var keys = Object.keys(add); + var i = keys.length; + while (i--) { + origin[keys[i]] = add[keys[i]]; + } + return origin; + } + function hasOwnProperty(obj, prop) { + return Object.prototype.hasOwnProperty.call(obj, prop); + } + + function BufferList() { + this.head = null; + this.tail = null; + this.length = 0; + } + + BufferList.prototype.push = function (v) { + var entry = { data: v, next: null }; + if (this.length > 0) this.tail.next = entry;else this.head = entry; + this.tail = entry; + ++this.length; + }; + + BufferList.prototype.unshift = function (v) { + var entry = { data: v, next: this.head }; + if (this.length === 0) this.tail = entry; + this.head = entry; + ++this.length; + }; + + BufferList.prototype.shift = function () { + if (this.length === 0) return; + var ret = this.head.data; + if (this.length === 1) this.head = this.tail = null;else this.head = this.head.next; + --this.length; + return ret; + }; + + BufferList.prototype.clear = function () { + this.head = this.tail = null; + this.length = 0; + }; + + BufferList.prototype.join = function (s) { + if (this.length === 0) return ''; + var p = this.head; + var ret = '' + p.data; + while (p = p.next) { + ret += s + p.data; + }return ret; + }; + + BufferList.prototype.concat = function (n) { + if (this.length === 0) return Buffer.alloc(0); + if (this.length === 1) return this.head.data; + var ret = Buffer.allocUnsafe(n >>> 0); + var p = this.head; + var i = 0; + while (p) { + p.data.copy(ret, i); + i += p.data.length; + p = p.next; + } + return ret; + }; + + // Copyright Joyent, Inc. and other Node contributors. + var isBufferEncoding = Buffer.isEncoding + || function(encoding) { + switch (encoding && encoding.toLowerCase()) { + case 'hex': case 'utf8': case 'utf-8': case 'ascii': case 'binary': case 'base64': case 'ucs2': case 'ucs-2': case 'utf16le': case 'utf-16le': case 'raw': return true; + default: return false; + } + }; + + + function assertEncoding(encoding) { + if (encoding && !isBufferEncoding(encoding)) { + throw new Error('Unknown encoding: ' + encoding); + } + } + + // StringDecoder provides an interface for efficiently splitting a series of + // buffers into a series of JS strings without breaking apart multi-byte + // characters. CESU-8 is handled as part of the UTF-8 encoding. + // + // @TODO Handling all encodings inside a single object makes it very difficult + // to reason about this code, so it should be split up in the future. + // @TODO There should be a utf8-strict encoding that rejects invalid UTF-8 code + // points as used by CESU-8. + function StringDecoder(encoding) { + this.encoding = (encoding || 'utf8').toLowerCase().replace(/[-_]/, ''); + assertEncoding(encoding); + switch (this.encoding) { + case 'utf8': + // CESU-8 represents each of Surrogate Pair by 3-bytes + this.surrogateSize = 3; + break; + case 'ucs2': + case 'utf16le': + // UTF-16 represents each of Surrogate Pair by 2-bytes + this.surrogateSize = 2; + this.detectIncompleteChar = utf16DetectIncompleteChar; + break; + case 'base64': + // Base-64 stores 3 bytes in 4 chars, and pads the remainder. + this.surrogateSize = 3; + this.detectIncompleteChar = base64DetectIncompleteChar; + break; + default: + this.write = passThroughWrite; + return; + } + + // Enough space to store all bytes of a single character. UTF-8 needs 4 + // bytes, but CESU-8 may require up to 6 (3 bytes per surrogate). + this.charBuffer = new Buffer(6); + // Number of bytes received for the current incomplete multi-byte character. + this.charReceived = 0; + // Number of bytes expected for the current incomplete multi-byte character. + this.charLength = 0; + } + + // write decodes the given buffer and returns it as JS string that is + // guaranteed to not contain any partial multi-byte characters. Any partial + // character found at the end of the buffer is buffered up, and will be + // returned when calling write again with the remaining bytes. + // + // Note: Converting a Buffer containing an orphan surrogate to a String + // currently works, but converting a String to a Buffer (via `new Buffer`, or + // Buffer#write) will replace incomplete surrogates with the unicode + // replacement character. See https://codereview.chromium.org/121173009/ . + StringDecoder.prototype.write = function(buffer) { + var charStr = ''; + // if our last write ended with an incomplete multibyte character + while (this.charLength) { + // determine how many remaining bytes this buffer has to offer for this char + var available = (buffer.length >= this.charLength - this.charReceived) ? + this.charLength - this.charReceived : + buffer.length; + + // add the new bytes to the char buffer + buffer.copy(this.charBuffer, this.charReceived, 0, available); + this.charReceived += available; + + if (this.charReceived < this.charLength) { + // still not enough chars in this buffer? wait for more ... + return ''; + } + + // remove bytes belonging to the current character from the buffer + buffer = buffer.slice(available, buffer.length); + + // get the character that was split + charStr = this.charBuffer.slice(0, this.charLength).toString(this.encoding); + + // CESU-8: lead surrogate (D800-DBFF) is also the incomplete character + var charCode = charStr.charCodeAt(charStr.length - 1); + if (charCode >= 0xD800 && charCode <= 0xDBFF) { + this.charLength += this.surrogateSize; + charStr = ''; + continue; + } + this.charReceived = this.charLength = 0; + + // if there are no more bytes in this buffer, just emit our char + if (buffer.length === 0) { + return charStr; + } + break; + } + + // determine and set charLength / charReceived + this.detectIncompleteChar(buffer); + + var end = buffer.length; + if (this.charLength) { + // buffer the incomplete character bytes we got + buffer.copy(this.charBuffer, 0, buffer.length - this.charReceived, end); + end -= this.charReceived; + } + + charStr += buffer.toString(this.encoding, 0, end); + + var end = charStr.length - 1; + var charCode = charStr.charCodeAt(end); + // CESU-8: lead surrogate (D800-DBFF) is also the incomplete character + if (charCode >= 0xD800 && charCode <= 0xDBFF) { + var size = this.surrogateSize; + this.charLength += size; + this.charReceived += size; + this.charBuffer.copy(this.charBuffer, size, 0, size); + buffer.copy(this.charBuffer, 0, 0, size); + return charStr.substring(0, end); + } + + // or just emit the charStr + return charStr; + }; + + // detectIncompleteChar determines if there is an incomplete UTF-8 character at + // the end of the given buffer. If so, it sets this.charLength to the byte + // length that character, and sets this.charReceived to the number of bytes + // that are available for this character. + StringDecoder.prototype.detectIncompleteChar = function(buffer) { + // determine how many bytes we have to check at the end of this buffer + var i = (buffer.length >= 3) ? 3 : buffer.length; + + // Figure out if one of the last i bytes of our buffer announces an + // incomplete char. + for (; i > 0; i--) { + var c = buffer[buffer.length - i]; + + // See http://en.wikipedia.org/wiki/UTF-8#Description + + // 110XXXXX + if (i == 1 && c >> 5 == 0x06) { + this.charLength = 2; + break; + } + + // 1110XXXX + if (i <= 2 && c >> 4 == 0x0E) { + this.charLength = 3; + break; + } + + // 11110XXX + if (i <= 3 && c >> 3 == 0x1E) { + this.charLength = 4; + break; + } + } + this.charReceived = i; + }; + + StringDecoder.prototype.end = function(buffer) { + var res = ''; + if (buffer && buffer.length) + res = this.write(buffer); + + if (this.charReceived) { + var cr = this.charReceived; + var buf = this.charBuffer; + var enc = this.encoding; + res += buf.slice(0, cr).toString(enc); + } + + return res; + }; + + function passThroughWrite(buffer) { + return buffer.toString(this.encoding); + } + + function utf16DetectIncompleteChar(buffer) { + this.charReceived = buffer.length % 2; + this.charLength = this.charReceived ? 2 : 0; + } + + function base64DetectIncompleteChar(buffer) { + this.charReceived = buffer.length % 3; + this.charLength = this.charReceived ? 3 : 0; + } + + Readable.ReadableState = ReadableState; + + var debug = debuglog('stream'); + inherits$1(Readable, EventEmitter); + + function prependListener(emitter, event, fn) { + // Sadly this is not cacheable as some libraries bundle their own + // event emitter implementation with them. + if (typeof emitter.prependListener === 'function') { + return emitter.prependListener(event, fn); + } else { + // This is a hack to make sure that our error handler is attached before any + // userland ones. NEVER DO THIS. This is here only because this code needs + // to continue to work with older versions of Node.js that do not include + // the prependListener() method. The goal is to eventually remove this hack. + if (!emitter._events || !emitter._events[event]) + emitter.on(event, fn); + else if (Array.isArray(emitter._events[event])) + emitter._events[event].unshift(fn); + else + emitter._events[event] = [fn, emitter._events[event]]; + } + } + function listenerCount (emitter, type) { + return emitter.listeners(type).length; + } + function ReadableState(options, stream) { + + options = options || {}; + + // object stream flag. Used to make read(n) ignore n and to + // make all the buffer merging and length checks go away + this.objectMode = !!options.objectMode; + + if (stream instanceof Duplex) this.objectMode = this.objectMode || !!options.readableObjectMode; + + // the point at which it stops calling _read() to fill the buffer + // Note: 0 is a valid value, means "don't call _read preemptively ever" + var hwm = options.highWaterMark; + var defaultHwm = this.objectMode ? 16 : 16 * 1024; + this.highWaterMark = hwm || hwm === 0 ? hwm : defaultHwm; + + // cast to ints. + this.highWaterMark = ~ ~this.highWaterMark; + + // A linked list is used to store data chunks instead of an array because the + // linked list can remove elements from the beginning faster than + // array.shift() + this.buffer = new BufferList(); + this.length = 0; + this.pipes = null; + this.pipesCount = 0; + this.flowing = null; + this.ended = false; + this.endEmitted = false; + this.reading = false; + + // a flag to be able to tell if the onwrite cb is called immediately, + // or on a later tick. We set this to true at first, because any + // actions that shouldn't happen until "later" should generally also + // not happen before the first write call. + this.sync = true; + + // whenever we return null, then we set a flag to say + // that we're awaiting a 'readable' event emission. + this.needReadable = false; + this.emittedReadable = false; + this.readableListening = false; + this.resumeScheduled = false; + + // Crypto is kind of old and crusty. Historically, its default string + // encoding is 'binary' so we have to make this configurable. + // Everything else in the universe uses 'utf8', though. + this.defaultEncoding = options.defaultEncoding || 'utf8'; + + // when piping, we only care about 'readable' events that happen + // after read()ing all the bytes and not getting any pushback. + this.ranOut = false; + + // the number of writers that are awaiting a drain event in .pipe()s + this.awaitDrain = 0; + + // if true, a maybeReadMore has been scheduled + this.readingMore = false; + + this.decoder = null; + this.encoding = null; + if (options.encoding) { + this.decoder = new StringDecoder(options.encoding); + this.encoding = options.encoding; + } + } + function Readable(options) { + + if (!(this instanceof Readable)) return new Readable(options); + + this._readableState = new ReadableState(options, this); + + // legacy + this.readable = true; + + if (options && typeof options.read === 'function') this._read = options.read; + + EventEmitter.call(this); + } + + // Manually shove something into the read() buffer. + // This returns true if the highWaterMark has not been hit yet, + // similar to how Writable.write() returns true if you should + // write() some more. + Readable.prototype.push = function (chunk, encoding) { + var state = this._readableState; + + if (!state.objectMode && typeof chunk === 'string') { + encoding = encoding || state.defaultEncoding; + if (encoding !== state.encoding) { + chunk = Buffer.from(chunk, encoding); + encoding = ''; + } + } + + return readableAddChunk(this, state, chunk, encoding, false); + }; + + // Unshift should *always* be something directly out of read() + Readable.prototype.unshift = function (chunk) { + var state = this._readableState; + return readableAddChunk(this, state, chunk, '', true); + }; + + Readable.prototype.isPaused = function () { + return this._readableState.flowing === false; + }; + + function readableAddChunk(stream, state, chunk, encoding, addToFront) { + var er = chunkInvalid(state, chunk); + if (er) { + stream.emit('error', er); + } else if (chunk === null) { + state.reading = false; + onEofChunk(stream, state); + } else if (state.objectMode || chunk && chunk.length > 0) { + if (state.ended && !addToFront) { + var e = new Error('stream.push() after EOF'); + stream.emit('error', e); + } else if (state.endEmitted && addToFront) { + var _e = new Error('stream.unshift() after end event'); + stream.emit('error', _e); + } else { + var skipAdd; + if (state.decoder && !addToFront && !encoding) { + chunk = state.decoder.write(chunk); + skipAdd = !state.objectMode && chunk.length === 0; + } + + if (!addToFront) state.reading = false; + + // Don't add to the buffer if we've decoded to an empty string chunk and + // we're not in object mode + if (!skipAdd) { + // if we want the data now, just emit it. + if (state.flowing && state.length === 0 && !state.sync) { + stream.emit('data', chunk); + stream.read(0); + } else { + // update the buffer info. + state.length += state.objectMode ? 1 : chunk.length; + if (addToFront) state.buffer.unshift(chunk);else state.buffer.push(chunk); + + if (state.needReadable) emitReadable(stream); + } + } + + maybeReadMore(stream, state); + } + } else if (!addToFront) { + state.reading = false; + } + + return needMoreData(state); + } + + // if it's past the high water mark, we can push in some more. + // Also, if we have no data yet, we can stand some + // more bytes. This is to work around cases where hwm=0, + // such as the repl. Also, if the push() triggered a + // readable event, and the user called read(largeNumber) such that + // needReadable was set, then we ought to push more, so that another + // 'readable' event will be triggered. + function needMoreData(state) { + return !state.ended && (state.needReadable || state.length < state.highWaterMark || state.length === 0); + } + + // backwards compatibility. + Readable.prototype.setEncoding = function (enc) { + this._readableState.decoder = new StringDecoder(enc); + this._readableState.encoding = enc; + return this; + }; + + // Don't raise the hwm > 8MB + var MAX_HWM = 0x800000; + function computeNewHighWaterMark(n) { + if (n >= MAX_HWM) { + n = MAX_HWM; + } else { + // Get the next highest power of 2 to prevent increasing hwm excessively in + // tiny amounts + n--; + n |= n >>> 1; + n |= n >>> 2; + n |= n >>> 4; + n |= n >>> 8; + n |= n >>> 16; + n++; + } + return n; + } + + // This function is designed to be inlinable, so please take care when making + // changes to the function body. + function howMuchToRead(n, state) { + if (n <= 0 || state.length === 0 && state.ended) return 0; + if (state.objectMode) return 1; + if (n !== n) { + // Only flow one buffer at a time + if (state.flowing && state.length) return state.buffer.head.data.length;else return state.length; + } + // If we're asking for more than the current hwm, then raise the hwm. + if (n > state.highWaterMark) state.highWaterMark = computeNewHighWaterMark(n); + if (n <= state.length) return n; + // Don't have enough + if (!state.ended) { + state.needReadable = true; + return 0; + } + return state.length; + } + + // you can override either this method, or the async _read(n) below. + Readable.prototype.read = function (n) { + debug('read', n); + n = parseInt(n, 10); + var state = this._readableState; + var nOrig = n; + + if (n !== 0) state.emittedReadable = false; + + // if we're doing read(0) to trigger a readable event, but we + // already have a bunch of data in the buffer, then just trigger + // the 'readable' event and move on. + if (n === 0 && state.needReadable && (state.length >= state.highWaterMark || state.ended)) { + debug('read: emitReadable', state.length, state.ended); + if (state.length === 0 && state.ended) endReadable(this);else emitReadable(this); + return null; + } + + n = howMuchToRead(n, state); + + // if we've ended, and we're now clear, then finish it up. + if (n === 0 && state.ended) { + if (state.length === 0) endReadable(this); + return null; + } + + // All the actual chunk generation logic needs to be + // *below* the call to _read. The reason is that in certain + // synthetic stream cases, such as passthrough streams, _read + // may be a completely synchronous operation which may change + // the state of the read buffer, providing enough data when + // before there was *not* enough. + // + // So, the steps are: + // 1. Figure out what the state of things will be after we do + // a read from the buffer. + // + // 2. If that resulting state will trigger a _read, then call _read. + // Note that this may be asynchronous, or synchronous. Yes, it is + // deeply ugly to write APIs this way, but that still doesn't mean + // that the Readable class should behave improperly, as streams are + // designed to be sync/async agnostic. + // Take note if the _read call is sync or async (ie, if the read call + // has returned yet), so that we know whether or not it's safe to emit + // 'readable' etc. + // + // 3. Actually pull the requested chunks out of the buffer and return. + + // if we need a readable event, then we need to do some reading. + var doRead = state.needReadable; + debug('need readable', doRead); + + // if we currently have less than the highWaterMark, then also read some + if (state.length === 0 || state.length - n < state.highWaterMark) { + doRead = true; + debug('length less than watermark', doRead); + } + + // however, if we've ended, then there's no point, and if we're already + // reading, then it's unnecessary. + if (state.ended || state.reading) { + doRead = false; + debug('reading or ended', doRead); + } else if (doRead) { + debug('do read'); + state.reading = true; + state.sync = true; + // if the length is currently zero, then we *need* a readable event. + if (state.length === 0) state.needReadable = true; + // call internal read method + this._read(state.highWaterMark); + state.sync = false; + // If _read pushed data synchronously, then `reading` will be false, + // and we need to re-evaluate how much data we can return to the user. + if (!state.reading) n = howMuchToRead(nOrig, state); + } + + var ret; + if (n > 0) ret = fromList(n, state);else ret = null; + + if (ret === null) { + state.needReadable = true; + n = 0; + } else { + state.length -= n; + } + + if (state.length === 0) { + // If we have nothing in the buffer, then we want to know + // as soon as we *do* get something into the buffer. + if (!state.ended) state.needReadable = true; + + // If we tried to read() past the EOF, then emit end on the next tick. + if (nOrig !== n && state.ended) endReadable(this); + } + + if (ret !== null) this.emit('data', ret); + + return ret; + }; + + function chunkInvalid(state, chunk) { + var er = null; + if (!isBuffer(chunk) && typeof chunk !== 'string' && chunk !== null && chunk !== undefined && !state.objectMode) { + er = new TypeError('Invalid non-string/buffer chunk'); + } + return er; + } + + function onEofChunk(stream, state) { + if (state.ended) return; + if (state.decoder) { + var chunk = state.decoder.end(); + if (chunk && chunk.length) { + state.buffer.push(chunk); + state.length += state.objectMode ? 1 : chunk.length; + } + } + state.ended = true; + + // emit 'readable' now to make sure it gets picked up. + emitReadable(stream); + } + + // Don't emit readable right away in sync mode, because this can trigger + // another read() call => stack overflow. This way, it might trigger + // a nextTick recursion warning, but that's not so bad. + function emitReadable(stream) { + var state = stream._readableState; + state.needReadable = false; + if (!state.emittedReadable) { + debug('emitReadable', state.flowing); + state.emittedReadable = true; + if (state.sync) nextTick(emitReadable_, stream);else emitReadable_(stream); + } + } + + function emitReadable_(stream) { + debug('emit readable'); + stream.emit('readable'); + flow(stream); + } + + // at this point, the user has presumably seen the 'readable' event, + // and called read() to consume some data. that may have triggered + // in turn another _read(n) call, in which case reading = true if + // it's in progress. + // However, if we're not ended, or reading, and the length < hwm, + // then go ahead and try to read some more preemptively. + function maybeReadMore(stream, state) { + if (!state.readingMore) { + state.readingMore = true; + nextTick(maybeReadMore_, stream, state); + } + } + + function maybeReadMore_(stream, state) { + var len = state.length; + while (!state.reading && !state.flowing && !state.ended && state.length < state.highWaterMark) { + debug('maybeReadMore read 0'); + stream.read(0); + if (len === state.length) + // didn't get any data, stop spinning. + break;else len = state.length; + } + state.readingMore = false; + } + + // abstract method. to be overridden in specific implementation classes. + // call cb(er, data) where data is <= n in length. + // for virtual (non-string, non-buffer) streams, "length" is somewhat + // arbitrary, and perhaps not very meaningful. + Readable.prototype._read = function (n) { + this.emit('error', new Error('not implemented')); + }; + + Readable.prototype.pipe = function (dest, pipeOpts) { + var src = this; + var state = this._readableState; + + switch (state.pipesCount) { + case 0: + state.pipes = dest; + break; + case 1: + state.pipes = [state.pipes, dest]; + break; + default: + state.pipes.push(dest); + break; + } + state.pipesCount += 1; + debug('pipe count=%d opts=%j', state.pipesCount, pipeOpts); + + var doEnd = (!pipeOpts || pipeOpts.end !== false); + + var endFn = doEnd ? onend : cleanup; + if (state.endEmitted) nextTick(endFn);else src.once('end', endFn); + + dest.on('unpipe', onunpipe); + function onunpipe(readable) { + debug('onunpipe'); + if (readable === src) { + cleanup(); + } + } + + function onend() { + debug('onend'); + dest.end(); + } + + // when the dest drains, it reduces the awaitDrain counter + // on the source. This would be more elegant with a .once() + // handler in flow(), but adding and removing repeatedly is + // too slow. + var ondrain = pipeOnDrain(src); + dest.on('drain', ondrain); + + var cleanedUp = false; + function cleanup() { + debug('cleanup'); + // cleanup event handlers once the pipe is broken + dest.removeListener('close', onclose); + dest.removeListener('finish', onfinish); + dest.removeListener('drain', ondrain); + dest.removeListener('error', onerror); + dest.removeListener('unpipe', onunpipe); + src.removeListener('end', onend); + src.removeListener('end', cleanup); + src.removeListener('data', ondata); + + cleanedUp = true; + + // if the reader is waiting for a drain event from this + // specific writer, then it would cause it to never start + // flowing again. + // So, if this is awaiting a drain, then we just call it now. + // If we don't know, then assume that we are waiting for one. + if (state.awaitDrain && (!dest._writableState || dest._writableState.needDrain)) ondrain(); + } + + // If the user pushes more data while we're writing to dest then we'll end up + // in ondata again. However, we only want to increase awaitDrain once because + // dest will only emit one 'drain' event for the multiple writes. + // => Introduce a guard on increasing awaitDrain. + var increasedAwaitDrain = false; + src.on('data', ondata); + function ondata(chunk) { + debug('ondata'); + increasedAwaitDrain = false; + var ret = dest.write(chunk); + if (false === ret && !increasedAwaitDrain) { + // If the user unpiped during `dest.write()`, it is possible + // to get stuck in a permanently paused state if that write + // also returned false. + // => Check whether `dest` is still a piping destination. + if ((state.pipesCount === 1 && state.pipes === dest || state.pipesCount > 1 && indexOf(state.pipes, dest) !== -1) && !cleanedUp) { + debug('false write response, pause', src._readableState.awaitDrain); + src._readableState.awaitDrain++; + increasedAwaitDrain = true; + } + src.pause(); + } + } + + // if the dest has an error, then stop piping into it. + // however, don't suppress the throwing behavior for this. + function onerror(er) { + debug('onerror', er); + unpipe(); + dest.removeListener('error', onerror); + if (listenerCount(dest, 'error') === 0) dest.emit('error', er); + } + + // Make sure our error handler is attached before userland ones. + prependListener(dest, 'error', onerror); + + // Both close and finish should trigger unpipe, but only once. + function onclose() { + dest.removeListener('finish', onfinish); + unpipe(); + } + dest.once('close', onclose); + function onfinish() { + debug('onfinish'); + dest.removeListener('close', onclose); + unpipe(); + } + dest.once('finish', onfinish); + + function unpipe() { + debug('unpipe'); + src.unpipe(dest); + } + + // tell the dest that it's being piped to + dest.emit('pipe', src); + + // start the flow if it hasn't been started already. + if (!state.flowing) { + debug('pipe resume'); + src.resume(); + } + + return dest; + }; + + function pipeOnDrain(src) { + return function () { + var state = src._readableState; + debug('pipeOnDrain', state.awaitDrain); + if (state.awaitDrain) state.awaitDrain--; + if (state.awaitDrain === 0 && src.listeners('data').length) { + state.flowing = true; + flow(src); + } + }; + } + + Readable.prototype.unpipe = function (dest) { + var state = this._readableState; + + // if we're not piping anywhere, then do nothing. + if (state.pipesCount === 0) return this; + + // just one destination. most common case. + if (state.pipesCount === 1) { + // passed in one, but it's not the right one. + if (dest && dest !== state.pipes) return this; + + if (!dest) dest = state.pipes; + + // got a match. + state.pipes = null; + state.pipesCount = 0; + state.flowing = false; + if (dest) dest.emit('unpipe', this); + return this; + } + + // slow case. multiple pipe destinations. + + if (!dest) { + // remove all. + var dests = state.pipes; + var len = state.pipesCount; + state.pipes = null; + state.pipesCount = 0; + state.flowing = false; + + for (var _i = 0; _i < len; _i++) { + dests[_i].emit('unpipe', this); + }return this; + } + + // try to find the right one. + var i = indexOf(state.pipes, dest); + if (i === -1) return this; + + state.pipes.splice(i, 1); + state.pipesCount -= 1; + if (state.pipesCount === 1) state.pipes = state.pipes[0]; + + dest.emit('unpipe', this); + + return this; + }; + + // set up data events if they are asked for + // Ensure readable listeners eventually get something + Readable.prototype.on = function (ev, fn) { + var res = EventEmitter.prototype.on.call(this, ev, fn); + + if (ev === 'data') { + // Start flowing on next tick if stream isn't explicitly paused + if (this._readableState.flowing !== false) this.resume(); + } else if (ev === 'readable') { + var state = this._readableState; + if (!state.endEmitted && !state.readableListening) { + state.readableListening = state.needReadable = true; + state.emittedReadable = false; + if (!state.reading) { + nextTick(nReadingNextTick, this); + } else if (state.length) { + emitReadable(this); + } + } + } + + return res; + }; + Readable.prototype.addListener = Readable.prototype.on; + + function nReadingNextTick(self) { + debug('readable nexttick read 0'); + self.read(0); + } + + // pause() and resume() are remnants of the legacy readable stream API + // If the user uses them, then switch into old mode. + Readable.prototype.resume = function () { + var state = this._readableState; + if (!state.flowing) { + debug('resume'); + state.flowing = true; + resume(this, state); + } + return this; + }; + + function resume(stream, state) { + if (!state.resumeScheduled) { + state.resumeScheduled = true; + nextTick(resume_, stream, state); + } + } + + function resume_(stream, state) { + if (!state.reading) { + debug('resume read 0'); + stream.read(0); + } + + state.resumeScheduled = false; + state.awaitDrain = 0; + stream.emit('resume'); + flow(stream); + if (state.flowing && !state.reading) stream.read(0); + } + + Readable.prototype.pause = function () { + debug('call pause flowing=%j', this._readableState.flowing); + if (false !== this._readableState.flowing) { + debug('pause'); + this._readableState.flowing = false; + this.emit('pause'); + } + return this; + }; + + function flow(stream) { + var state = stream._readableState; + debug('flow', state.flowing); + while (state.flowing && stream.read() !== null) {} + } + + // wrap an old-style stream as the async data source. + // This is *not* part of the readable stream interface. + // It is an ugly unfortunate mess of history. + Readable.prototype.wrap = function (stream) { + var state = this._readableState; + var paused = false; + + var self = this; + stream.on('end', function () { + debug('wrapped end'); + if (state.decoder && !state.ended) { + var chunk = state.decoder.end(); + if (chunk && chunk.length) self.push(chunk); + } + + self.push(null); + }); + + stream.on('data', function (chunk) { + debug('wrapped data'); + if (state.decoder) chunk = state.decoder.write(chunk); + + // don't skip over falsy values in objectMode + if (state.objectMode && (chunk === null || chunk === undefined)) return;else if (!state.objectMode && (!chunk || !chunk.length)) return; + + var ret = self.push(chunk); + if (!ret) { + paused = true; + stream.pause(); + } + }); + + // proxy all the other methods. + // important when wrapping filters and duplexes. + for (var i in stream) { + if (this[i] === undefined && typeof stream[i] === 'function') { + this[i] = function (method) { + return function () { + return stream[method].apply(stream, arguments); + }; + }(i); + } + } + + // proxy certain important events. + var events = ['error', 'close', 'destroy', 'pause', 'resume']; + forEach(events, function (ev) { + stream.on(ev, self.emit.bind(self, ev)); + }); + + // when we try to consume some more bytes, simply unpause the + // underlying stream. + self._read = function (n) { + debug('wrapped _read', n); + if (paused) { + paused = false; + stream.resume(); + } + }; + + return self; + }; + + // exposed for testing purposes only. + Readable._fromList = fromList; + + // Pluck off n bytes from an array of buffers. + // Length is the combined lengths of all the buffers in the list. + // This function is designed to be inlinable, so please take care when making + // changes to the function body. + function fromList(n, state) { + // nothing buffered + if (state.length === 0) return null; + + var ret; + if (state.objectMode) ret = state.buffer.shift();else if (!n || n >= state.length) { + // read it all, truncate the list + if (state.decoder) ret = state.buffer.join('');else if (state.buffer.length === 1) ret = state.buffer.head.data;else ret = state.buffer.concat(state.length); + state.buffer.clear(); + } else { + // read part of list + ret = fromListPartial(n, state.buffer, state.decoder); + } + + return ret; + } + + // Extracts only enough buffered data to satisfy the amount requested. + // This function is designed to be inlinable, so please take care when making + // changes to the function body. + function fromListPartial(n, list, hasStrings) { + var ret; + if (n < list.head.data.length) { + // slice is the same for buffers and strings + ret = list.head.data.slice(0, n); + list.head.data = list.head.data.slice(n); + } else if (n === list.head.data.length) { + // first chunk is a perfect match + ret = list.shift(); + } else { + // result spans more than one buffer + ret = hasStrings ? copyFromBufferString(n, list) : copyFromBuffer(n, list); + } + return ret; + } + + // Copies a specified amount of characters from the list of buffered data + // chunks. + // This function is designed to be inlinable, so please take care when making + // changes to the function body. + function copyFromBufferString(n, list) { + var p = list.head; + var c = 1; + var ret = p.data; + n -= ret.length; + while (p = p.next) { + var str = p.data; + var nb = n > str.length ? str.length : n; + if (nb === str.length) ret += str;else ret += str.slice(0, n); + n -= nb; + if (n === 0) { + if (nb === str.length) { + ++c; + if (p.next) list.head = p.next;else list.head = list.tail = null; + } else { + list.head = p; + p.data = str.slice(nb); + } + break; + } + ++c; + } + list.length -= c; + return ret; + } + + // Copies a specified amount of bytes from the list of buffered data chunks. + // This function is designed to be inlinable, so please take care when making + // changes to the function body. + function copyFromBuffer(n, list) { + var ret = Buffer.allocUnsafe(n); + var p = list.head; + var c = 1; + p.data.copy(ret); + n -= p.data.length; + while (p = p.next) { + var buf = p.data; + var nb = n > buf.length ? buf.length : n; + buf.copy(ret, ret.length - n, 0, nb); + n -= nb; + if (n === 0) { + if (nb === buf.length) { + ++c; + if (p.next) list.head = p.next;else list.head = list.tail = null; + } else { + list.head = p; + p.data = buf.slice(nb); + } + break; + } + ++c; + } + list.length -= c; + return ret; + } + + function endReadable(stream) { + var state = stream._readableState; + + // If we get here before consuming all the bytes, then that is a + // bug in node. Should never happen. + if (state.length > 0) throw new Error('"endReadable()" called on non-empty stream'); + + if (!state.endEmitted) { + state.ended = true; + nextTick(endReadableNT, state, stream); + } + } + + function endReadableNT(state, stream) { + // Check that we didn't get one last unshift. + if (!state.endEmitted && state.length === 0) { + state.endEmitted = true; + stream.readable = false; + stream.emit('end'); + } + } + + function forEach(xs, f) { + for (var i = 0, l = xs.length; i < l; i++) { + f(xs[i], i); + } + } + + function indexOf(xs, x) { + for (var i = 0, l = xs.length; i < l; i++) { + if (xs[i] === x) return i; + } + return -1; + } + + // A bit simpler than readable streams. + Writable.WritableState = WritableState; + inherits$1(Writable, EventEmitter); + + function nop() {} + + function WriteReq(chunk, encoding, cb) { + this.chunk = chunk; + this.encoding = encoding; + this.callback = cb; + this.next = null; + } + + function WritableState(options, stream) { + Object.defineProperty(this, 'buffer', { + get: deprecate(function () { + return this.getBuffer(); + }, '_writableState.buffer is deprecated. Use _writableState.getBuffer ' + 'instead.') + }); + options = options || {}; + + // object stream flag to indicate whether or not this stream + // contains buffers or objects. + this.objectMode = !!options.objectMode; + + if (stream instanceof Duplex) this.objectMode = this.objectMode || !!options.writableObjectMode; + + // the point at which write() starts returning false + // Note: 0 is a valid value, means that we always return false if + // the entire buffer is not flushed immediately on write() + var hwm = options.highWaterMark; + var defaultHwm = this.objectMode ? 16 : 16 * 1024; + this.highWaterMark = hwm || hwm === 0 ? hwm : defaultHwm; + + // cast to ints. + this.highWaterMark = ~ ~this.highWaterMark; + + this.needDrain = false; + // at the start of calling end() + this.ending = false; + // when end() has been called, and returned + this.ended = false; + // when 'finish' is emitted + this.finished = false; + + // should we decode strings into buffers before passing to _write? + // this is here so that some node-core streams can optimize string + // handling at a lower level. + var noDecode = options.decodeStrings === false; + this.decodeStrings = !noDecode; + + // Crypto is kind of old and crusty. Historically, its default string + // encoding is 'binary' so we have to make this configurable. + // Everything else in the universe uses 'utf8', though. + this.defaultEncoding = options.defaultEncoding || 'utf8'; + + // not an actual buffer we keep track of, but a measurement + // of how much we're waiting to get pushed to some underlying + // socket or file. + this.length = 0; + + // a flag to see when we're in the middle of a write. + this.writing = false; + + // when true all writes will be buffered until .uncork() call + this.corked = 0; + + // a flag to be able to tell if the onwrite cb is called immediately, + // or on a later tick. We set this to true at first, because any + // actions that shouldn't happen until "later" should generally also + // not happen before the first write call. + this.sync = true; + + // a flag to know if we're processing previously buffered items, which + // may call the _write() callback in the same tick, so that we don't + // end up in an overlapped onwrite situation. + this.bufferProcessing = false; + + // the callback that's passed to _write(chunk,cb) + this.onwrite = function (er) { + onwrite(stream, er); + }; + + // the callback that the user supplies to write(chunk,encoding,cb) + this.writecb = null; + + // the amount that is being written when _write is called. + this.writelen = 0; + + this.bufferedRequest = null; + this.lastBufferedRequest = null; + + // number of pending user-supplied write callbacks + // this must be 0 before 'finish' can be emitted + this.pendingcb = 0; + + // emit prefinish if the only thing we're waiting for is _write cbs + // This is relevant for synchronous Transform streams + this.prefinished = false; + + // True if the error was already emitted and should not be thrown again + this.errorEmitted = false; + + // count buffered requests + this.bufferedRequestCount = 0; + + // allocate the first CorkedRequest, there is always + // one allocated and free to use, and we maintain at most two + this.corkedRequestsFree = new CorkedRequest(this); + } + + WritableState.prototype.getBuffer = function writableStateGetBuffer() { + var current = this.bufferedRequest; + var out = []; + while (current) { + out.push(current); + current = current.next; + } + return out; + }; + function Writable(options) { + + // Writable ctor is applied to Duplexes, though they're not + // instanceof Writable, they're instanceof Readable. + if (!(this instanceof Writable) && !(this instanceof Duplex)) return new Writable(options); + + this._writableState = new WritableState(options, this); + + // legacy. + this.writable = true; + + if (options) { + if (typeof options.write === 'function') this._write = options.write; + + if (typeof options.writev === 'function') this._writev = options.writev; + } + + EventEmitter.call(this); + } + + // Otherwise people can pipe Writable streams, which is just wrong. + Writable.prototype.pipe = function () { + this.emit('error', new Error('Cannot pipe, not readable')); + }; + + function writeAfterEnd(stream, cb) { + var er = new Error('write after end'); + // TODO: defer error events consistently everywhere, not just the cb + stream.emit('error', er); + nextTick(cb, er); + } + + // If we get something that is not a buffer, string, null, or undefined, + // and we're not in objectMode, then that's an error. + // Otherwise stream chunks are all considered to be of length=1, and the + // watermarks determine how many objects to keep in the buffer, rather than + // how many bytes or characters. + function validChunk(stream, state, chunk, cb) { + var valid = true; + var er = false; + // Always throw error if a null is written + // if we are not in object mode then throw + // if it is not a buffer, string, or undefined. + if (chunk === null) { + er = new TypeError('May not write null values to stream'); + } else if (!Buffer.isBuffer(chunk) && typeof chunk !== 'string' && chunk !== undefined && !state.objectMode) { + er = new TypeError('Invalid non-string/buffer chunk'); + } + if (er) { + stream.emit('error', er); + nextTick(cb, er); + valid = false; + } + return valid; + } + + Writable.prototype.write = function (chunk, encoding, cb) { + var state = this._writableState; + var ret = false; + + if (typeof encoding === 'function') { + cb = encoding; + encoding = null; + } + + if (Buffer.isBuffer(chunk)) encoding = 'buffer';else if (!encoding) encoding = state.defaultEncoding; + + if (typeof cb !== 'function') cb = nop; + + if (state.ended) writeAfterEnd(this, cb);else if (validChunk(this, state, chunk, cb)) { + state.pendingcb++; + ret = writeOrBuffer(this, state, chunk, encoding, cb); + } + + return ret; + }; + + Writable.prototype.cork = function () { + var state = this._writableState; + + state.corked++; + }; + + Writable.prototype.uncork = function () { + var state = this._writableState; + + if (state.corked) { + state.corked--; + + if (!state.writing && !state.corked && !state.finished && !state.bufferProcessing && state.bufferedRequest) clearBuffer(this, state); + } + }; + + Writable.prototype.setDefaultEncoding = function setDefaultEncoding(encoding) { + // node::ParseEncoding() requires lower case. + if (typeof encoding === 'string') encoding = encoding.toLowerCase(); + if (!(['hex', 'utf8', 'utf-8', 'ascii', 'binary', 'base64', 'ucs2', 'ucs-2', 'utf16le', 'utf-16le', 'raw'].indexOf((encoding + '').toLowerCase()) > -1)) throw new TypeError('Unknown encoding: ' + encoding); + this._writableState.defaultEncoding = encoding; + return this; + }; + + function decodeChunk(state, chunk, encoding) { + if (!state.objectMode && state.decodeStrings !== false && typeof chunk === 'string') { + chunk = Buffer.from(chunk, encoding); + } + return chunk; + } + + // if we're already writing something, then just put this + // in the queue, and wait our turn. Otherwise, call _write + // If we return false, then we need a drain event, so set that flag. + function writeOrBuffer(stream, state, chunk, encoding, cb) { + chunk = decodeChunk(state, chunk, encoding); + + if (Buffer.isBuffer(chunk)) encoding = 'buffer'; + var len = state.objectMode ? 1 : chunk.length; + + state.length += len; + + var ret = state.length < state.highWaterMark; + // we must ensure that previous needDrain will not be reset to false. + if (!ret) state.needDrain = true; + + if (state.writing || state.corked) { + var last = state.lastBufferedRequest; + state.lastBufferedRequest = new WriteReq(chunk, encoding, cb); + if (last) { + last.next = state.lastBufferedRequest; + } else { + state.bufferedRequest = state.lastBufferedRequest; + } + state.bufferedRequestCount += 1; + } else { + doWrite(stream, state, false, len, chunk, encoding, cb); + } + + return ret; + } + + function doWrite(stream, state, writev, len, chunk, encoding, cb) { + state.writelen = len; + state.writecb = cb; + state.writing = true; + state.sync = true; + if (writev) stream._writev(chunk, state.onwrite);else stream._write(chunk, encoding, state.onwrite); + state.sync = false; + } + + function onwriteError(stream, state, sync, er, cb) { + --state.pendingcb; + if (sync) nextTick(cb, er);else cb(er); + + stream._writableState.errorEmitted = true; + stream.emit('error', er); + } + + function onwriteStateUpdate(state) { + state.writing = false; + state.writecb = null; + state.length -= state.writelen; + state.writelen = 0; + } + + function onwrite(stream, er) { + var state = stream._writableState; + var sync = state.sync; + var cb = state.writecb; + + onwriteStateUpdate(state); + + if (er) onwriteError(stream, state, sync, er, cb);else { + // Check if we're actually ready to finish, but don't emit yet + var finished = needFinish(state); + + if (!finished && !state.corked && !state.bufferProcessing && state.bufferedRequest) { + clearBuffer(stream, state); + } + + if (sync) { + /**/ + nextTick(afterWrite, stream, state, finished, cb); + /**/ + } else { + afterWrite(stream, state, finished, cb); + } + } + } + + function afterWrite(stream, state, finished, cb) { + if (!finished) onwriteDrain(stream, state); + state.pendingcb--; + cb(); + finishMaybe(stream, state); + } + + // Must force callback to be called on nextTick, so that we don't + // emit 'drain' before the write() consumer gets the 'false' return + // value, and has a chance to attach a 'drain' listener. + function onwriteDrain(stream, state) { + if (state.length === 0 && state.needDrain) { + state.needDrain = false; + stream.emit('drain'); + } + } + + // if there's something in the buffer waiting, then process it + function clearBuffer(stream, state) { + state.bufferProcessing = true; + var entry = state.bufferedRequest; + + if (stream._writev && entry && entry.next) { + // Fast case, write everything using _writev() + var l = state.bufferedRequestCount; + var buffer = new Array(l); + var holder = state.corkedRequestsFree; + holder.entry = entry; + + var count = 0; + while (entry) { + buffer[count] = entry; + entry = entry.next; + count += 1; + } + + doWrite(stream, state, true, state.length, buffer, '', holder.finish); + + // doWrite is almost always async, defer these to save a bit of time + // as the hot path ends with doWrite + state.pendingcb++; + state.lastBufferedRequest = null; + if (holder.next) { + state.corkedRequestsFree = holder.next; + holder.next = null; + } else { + state.corkedRequestsFree = new CorkedRequest(state); + } + } else { + // Slow case, write chunks one-by-one + while (entry) { + var chunk = entry.chunk; + var encoding = entry.encoding; + var cb = entry.callback; + var len = state.objectMode ? 1 : chunk.length; + + doWrite(stream, state, false, len, chunk, encoding, cb); + entry = entry.next; + // if we didn't call the onwrite immediately, then + // it means that we need to wait until it does. + // also, that means that the chunk and cb are currently + // being processed, so move the buffer counter past them. + if (state.writing) { + break; + } + } + + if (entry === null) state.lastBufferedRequest = null; + } + + state.bufferedRequestCount = 0; + state.bufferedRequest = entry; + state.bufferProcessing = false; + } + + Writable.prototype._write = function (chunk, encoding, cb) { + cb(new Error('not implemented')); + }; + + Writable.prototype._writev = null; + + Writable.prototype.end = function (chunk, encoding, cb) { + var state = this._writableState; + + if (typeof chunk === 'function') { + cb = chunk; + chunk = null; + encoding = null; + } else if (typeof encoding === 'function') { + cb = encoding; + encoding = null; + } + + if (chunk !== null && chunk !== undefined) this.write(chunk, encoding); + + // .end() fully uncorks + if (state.corked) { + state.corked = 1; + this.uncork(); + } + + // ignore unnecessary end() calls. + if (!state.ending && !state.finished) endWritable(this, state, cb); + }; + + function needFinish(state) { + return state.ending && state.length === 0 && state.bufferedRequest === null && !state.finished && !state.writing; + } + + function prefinish(stream, state) { + if (!state.prefinished) { + state.prefinished = true; + stream.emit('prefinish'); + } + } + + function finishMaybe(stream, state) { + var need = needFinish(state); + if (need) { + if (state.pendingcb === 0) { + prefinish(stream, state); + state.finished = true; + stream.emit('finish'); + } else { + prefinish(stream, state); + } + } + return need; + } + + function endWritable(stream, state, cb) { + state.ending = true; + finishMaybe(stream, state); + if (cb) { + if (state.finished) nextTick(cb);else stream.once('finish', cb); + } + state.ended = true; + stream.writable = false; + } + + // It seems a linked list but it is not + // there will be only 2 of these for each stream + function CorkedRequest(state) { + var _this = this; + + this.next = null; + this.entry = null; + + this.finish = function (err) { + var entry = _this.entry; + _this.entry = null; + while (entry) { + var cb = entry.callback; + state.pendingcb--; + cb(err); + entry = entry.next; + } + if (state.corkedRequestsFree) { + state.corkedRequestsFree.next = _this; + } else { + state.corkedRequestsFree = _this; + } + }; + } + + inherits$1(Duplex, Readable); + + var keys = Object.keys(Writable.prototype); + for (var v = 0; v < keys.length; v++) { + var method = keys[v]; + if (!Duplex.prototype[method]) Duplex.prototype[method] = Writable.prototype[method]; + } + function Duplex(options) { + if (!(this instanceof Duplex)) return new Duplex(options); + + Readable.call(this, options); + Writable.call(this, options); + + if (options && options.readable === false) this.readable = false; + + if (options && options.writable === false) this.writable = false; + + this.allowHalfOpen = true; + if (options && options.allowHalfOpen === false) this.allowHalfOpen = false; + + this.once('end', onend); + } + + // the no-half-open enforcer + function onend() { + // if we allow half-open state, or if the writable side ended, + // then we're ok. + if (this.allowHalfOpen || this._writableState.ended) return; + + // no more data can be written. + // But allow more writes to happen in this tick. + nextTick(onEndNT, this); + } + + function onEndNT(self) { + self.end(); + } + + // a transform stream is a readable/writable stream where you do + inherits$1(Transform, Duplex); + + function TransformState(stream) { + this.afterTransform = function (er, data) { + return afterTransform(stream, er, data); + }; + + this.needTransform = false; + this.transforming = false; + this.writecb = null; + this.writechunk = null; + this.writeencoding = null; + } + + function afterTransform(stream, er, data) { + var ts = stream._transformState; + ts.transforming = false; + + var cb = ts.writecb; + + if (!cb) return stream.emit('error', new Error('no writecb in Transform class')); + + ts.writechunk = null; + ts.writecb = null; + + if (data !== null && data !== undefined) stream.push(data); + + cb(er); + + var rs = stream._readableState; + rs.reading = false; + if (rs.needReadable || rs.length < rs.highWaterMark) { + stream._read(rs.highWaterMark); + } + } + function Transform(options) { + if (!(this instanceof Transform)) return new Transform(options); + + Duplex.call(this, options); + + this._transformState = new TransformState(this); + + // when the writable side finishes, then flush out anything remaining. + var stream = this; + + // start out asking for a readable event once data is transformed. + this._readableState.needReadable = true; + + // we have implemented the _read method, and done the other things + // that Readable wants before the first _read call, so unset the + // sync guard flag. + this._readableState.sync = false; + + if (options) { + if (typeof options.transform === 'function') this._transform = options.transform; + + if (typeof options.flush === 'function') this._flush = options.flush; + } + + this.once('prefinish', function () { + if (typeof this._flush === 'function') this._flush(function (er) { + done(stream, er); + });else done(stream); + }); + } + + Transform.prototype.push = function (chunk, encoding) { + this._transformState.needTransform = false; + return Duplex.prototype.push.call(this, chunk, encoding); + }; + + // This is the part where you do stuff! + // override this function in implementation classes. + // 'chunk' is an input chunk. + // + // Call `push(newChunk)` to pass along transformed output + // to the readable side. You may call 'push' zero or more times. + // + // Call `cb(err)` when you are done with this chunk. If you pass + // an error, then that'll put the hurt on the whole operation. If you + // never call cb(), then you'll never get another chunk. + Transform.prototype._transform = function (chunk, encoding, cb) { + throw new Error('Not implemented'); + }; + + Transform.prototype._write = function (chunk, encoding, cb) { + var ts = this._transformState; + ts.writecb = cb; + ts.writechunk = chunk; + ts.writeencoding = encoding; + if (!ts.transforming) { + var rs = this._readableState; + if (ts.needTransform || rs.needReadable || rs.length < rs.highWaterMark) this._read(rs.highWaterMark); + } + }; + + // Doesn't matter what the args are here. + // _transform does all the work. + // That we got here means that the readable side wants more data. + Transform.prototype._read = function (n) { + var ts = this._transformState; + + if (ts.writechunk !== null && ts.writecb && !ts.transforming) { + ts.transforming = true; + this._transform(ts.writechunk, ts.writeencoding, ts.afterTransform); + } else { + // mark that we need a transform, so that any data that comes in + // will get processed, now that we've asked for it. + ts.needTransform = true; + } + }; + + function done(stream, er) { + if (er) return stream.emit('error', er); + + // if there's nothing in the write buffer, then that means + // that nothing more will ever be provided + var ws = stream._writableState; + var ts = stream._transformState; + + if (ws.length) throw new Error('Calling transform done when ws.length != 0'); + + if (ts.transforming) throw new Error('Calling transform done when still transforming'); + + return stream.push(null); + } + + inherits$1(PassThrough, Transform); + function PassThrough(options) { + if (!(this instanceof PassThrough)) return new PassThrough(options); + + Transform.call(this, options); + } + + PassThrough.prototype._transform = function (chunk, encoding, cb) { + cb(null, chunk); + }; + + inherits$1(Stream, EventEmitter); + Stream.Readable = Readable; + Stream.Writable = Writable; + Stream.Duplex = Duplex; + Stream.Transform = Transform; + Stream.PassThrough = PassThrough; + + // Backwards-compat with node 0.4.x + Stream.Stream = Stream; + + // old-style streams. Note that the pipe method (the only relevant + // part of this class) is overridden in the Readable class. + + function Stream() { + EventEmitter.call(this); + } + + Stream.prototype.pipe = function(dest, options) { + var source = this; + + function ondata(chunk) { + if (dest.writable) { + if (false === dest.write(chunk) && source.pause) { + source.pause(); + } + } + } + + source.on('data', ondata); + + function ondrain() { + if (source.readable && source.resume) { + source.resume(); + } + } + + dest.on('drain', ondrain); + + // If the 'end' option is not supplied, dest.end() will be called when + // source gets the 'end' or 'close' events. Only dest.end() once. + if (!dest._isStdio && (!options || options.end !== false)) { + source.on('end', onend); + source.on('close', onclose); + } + + var didOnEnd = false; + function onend() { + if (didOnEnd) return; + didOnEnd = true; + + dest.end(); + } + + + function onclose() { + if (didOnEnd) return; + didOnEnd = true; + + if (typeof dest.destroy === 'function') dest.destroy(); + } + + // don't leave dangling pipes when there are errors. + function onerror(er) { + cleanup(); + if (EventEmitter.listenerCount(this, 'error') === 0) { + throw er; // Unhandled stream error in pipe. + } + } + + source.on('error', onerror); + dest.on('error', onerror); + + // remove all the event listeners that were added. + function cleanup() { + source.removeListener('data', ondata); + dest.removeListener('drain', ondrain); + + source.removeListener('end', onend); + source.removeListener('close', onclose); + + source.removeListener('error', onerror); + dest.removeListener('error', onerror); + + source.removeListener('end', cleanup); + source.removeListener('close', cleanup); + + dest.removeListener('close', cleanup); + } + + source.on('end', cleanup); + source.on('close', cleanup); + + dest.on('close', cleanup); + + dest.emit('pipe', source); + + // Allow for unix-like usage: A.pipe(B).pipe(C) + return dest; + }; + + class CsvError extends Error { + constructor(code, message, ...contexts) { + if(Array.isArray(message)) message = message.join(' '); + super(message); + if(Error.captureStackTrace !== undefined){ + Error.captureStackTrace(this, CsvError); + } + this.code = code; + for(const context of contexts){ + for(const key in context){ + const value = context[key]; + this[key] = isBuffer(value) ? value.toString() : value == null ? value : JSON.parse(JSON.stringify(value)); + } + } + } + } + + const is_object = function(obj){ + return typeof obj === 'object' && obj !== null && ! Array.isArray(obj); + }; + + // Lodash implementation of `get` + + const charCodeOfDot = '.'.charCodeAt(0); + const reEscapeChar = /\\(\\)?/g; + const rePropName = RegExp( + // Match anything that isn't a dot or bracket. + '[^.[\\]]+' + '|' + + // Or match property names within brackets. + '\\[(?:' + + // Match a non-string expression. + '([^"\'][^[]*)' + '|' + + // Or match strings (supports escaping characters). + '(["\'])((?:(?!\\2)[^\\\\]|\\\\.)*?)\\2' + + ')\\]'+ '|' + + // Or match "" as the space between consecutive dots or empty brackets. + '(?=(?:\\.|\\[\\])(?:\\.|\\[\\]|$))' + , 'g'); + const reIsDeepProp = /\.|\[(?:[^[\]]*|(["'])(?:(?!\1)[^\\]|\\.)*?\1)\]/; + const reIsPlainProp = /^\w*$/; + const getTag = function(value){ + return Object.prototype.toString.call(value); + }; + const isSymbol = function(value){ + const type = typeof value; + return type === 'symbol' || (type === 'object' && value && getTag(value) === '[object Symbol]'); + }; + const isKey = function(value, object){ + if(Array.isArray(value)){ + return false; + } + const type = typeof value; + if(type === 'number' || type === 'symbol' || type === 'boolean' || !value || isSymbol(value)){ + return true; + } + return reIsPlainProp.test(value) || !reIsDeepProp.test(value) || + (object != null && value in Object(object)); + }; + const stringToPath = function(string){ + const result = []; + if(string.charCodeAt(0) === charCodeOfDot){ + result.push(''); + } + string.replace(rePropName, function(match, expression, quote, subString){ + let key = match; + if(quote){ + key = subString.replace(reEscapeChar, '$1'); + }else if(expression){ + key = expression.trim(); + } + result.push(key); + }); + return result; + }; + const castPath = function(value, object){ + if(Array.isArray(value)){ + return value; + } else { + return isKey(value, object) ? [value] : stringToPath(value); + } + }; + const toKey = function(value){ + if(typeof value === 'string' || isSymbol(value)) + return value; + const result = `${value}`; + // eslint-disable-next-line + return (result == '0' && (1 / value) == -INFINITY) ? '-0' : result; + }; + const get = function(object, path){ + path = castPath(path, object); + let index = 0; + const length = path.length; + while(object != null && index < length){ + object = object[toKey(path[index++])]; + } + return (index && index === length) ? object : undefined; + }; + + const normalize_columns = function(columns){ + if(columns === undefined || columns === null){ + return [undefined, undefined]; + } + if(typeof columns !== 'object'){ + return [Error('Invalid option "columns": expect an array or an object')]; + } + if(!Array.isArray(columns)){ + const newcolumns = []; + for(const k in columns){ + newcolumns.push({ + key: k, + header: columns[k] + }); + } + columns = newcolumns; + }else { + const newcolumns = []; + for(const column of columns){ + if(typeof column === 'string'){ + newcolumns.push({ + key: column, + header: column + }); + }else if(typeof column === 'object' && column !== null && !Array.isArray(column)){ + if(!column.key){ + return [Error('Invalid column definition: property "key" is required')]; + } + if(column.header === undefined){ + column.header = column.key; + } + newcolumns.push(column); + }else { + return [Error('Invalid column definition: expect a string or an object')]; + } + } + columns = newcolumns; + } + return [undefined, columns]; + }; + + const underscore = function(str){ + return str.replace(/([A-Z])/g, function(_, match){ + return '_' + match.toLowerCase(); + }); + }; + + const normalize_options = function(opts) { + const options = {}; + // Merge with user options + for(const opt in opts){ + options[underscore(opt)] = opts[opt]; + } + // Normalize option `bom` + if(options.bom === undefined || options.bom === null || options.bom === false){ + options.bom = false; + }else if(options.bom !== true){ + return [new CsvError('CSV_OPTION_BOOLEAN_INVALID_TYPE', [ + 'option `bom` is optional and must be a boolean value,', + `got ${JSON.stringify(options.bom)}` + ])]; + } + // Normalize option `delimiter` + if(options.delimiter === undefined || options.delimiter === null){ + options.delimiter = ','; + }else if(isBuffer(options.delimiter)){ + options.delimiter = options.delimiter.toString(); + }else if(typeof options.delimiter !== 'string'){ + return [new CsvError('CSV_OPTION_DELIMITER_INVALID_TYPE', [ + 'option `delimiter` must be a buffer or a string,', + `got ${JSON.stringify(options.delimiter)}` + ])]; + } + // Normalize option `quote` + if(options.quote === undefined || options.quote === null){ + options.quote = '"'; + }else if(options.quote === true){ + options.quote = '"'; + }else if(options.quote === false){ + options.quote = ''; + }else if (isBuffer(options.quote)){ + options.quote = options.quote.toString(); + }else if(typeof options.quote !== 'string'){ + return [new CsvError('CSV_OPTION_QUOTE_INVALID_TYPE', [ + 'option `quote` must be a boolean, a buffer or a string,', + `got ${JSON.stringify(options.quote)}` + ])]; + } + // Normalize option `quoted` + if(options.quoted === undefined || options.quoted === null){ + options.quoted = false; + } + // Normalize option `quoted_empty` + if(options.quoted_empty === undefined || options.quoted_empty === null){ + options.quoted_empty = undefined; + } + // Normalize option `quoted_match` + if(options.quoted_match === undefined || options.quoted_match === null || options.quoted_match === false){ + options.quoted_match = null; + }else if(!Array.isArray(options.quoted_match)){ + options.quoted_match = [options.quoted_match]; + } + if(options.quoted_match){ + for(const quoted_match of options.quoted_match){ + const isString = typeof quoted_match === 'string'; + const isRegExp = quoted_match instanceof RegExp; + if(!isString && !isRegExp){ + return [Error(`Invalid Option: quoted_match must be a string or a regex, got ${JSON.stringify(quoted_match)}`)]; + } + } + } + // Normalize option `quoted_string` + if(options.quoted_string === undefined || options.quoted_string === null){ + options.quoted_string = false; + } + // Normalize option `eof` + if(options.eof === undefined || options.eof === null){ + options.eof = true; + } + // Normalize option `escape` + if(options.escape === undefined || options.escape === null){ + options.escape = '"'; + }else if(isBuffer(options.escape)){ + options.escape = options.escape.toString(); + }else if(typeof options.escape !== 'string'){ + return [Error(`Invalid Option: escape must be a buffer or a string, got ${JSON.stringify(options.escape)}`)]; + } + if (options.escape.length > 1){ + return [Error(`Invalid Option: escape must be one character, got ${options.escape.length} characters`)]; + } + // Normalize option `header` + if(options.header === undefined || options.header === null){ + options.header = false; + } + // Normalize option `columns` + const [errColumns, columns] = normalize_columns(options.columns); + if(errColumns !== undefined) return [errColumns]; + options.columns = columns; + // Normalize option `quoted` + if(options.quoted === undefined || options.quoted === null){ + options.quoted = false; + } + // Normalize option `cast` + if(options.cast === undefined || options.cast === null){ + options.cast = {}; + } + // Normalize option cast.bigint + if(options.cast.bigint === undefined || options.cast.bigint === null){ + // Cast boolean to string by default + options.cast.bigint = value => '' + value; + } + // Normalize option cast.boolean + if(options.cast.boolean === undefined || options.cast.boolean === null){ + // Cast boolean to string by default + options.cast.boolean = value => value ? '1' : ''; + } + // Normalize option cast.date + if(options.cast.date === undefined || options.cast.date === null){ + // Cast date to timestamp string by default + options.cast.date = value => '' + value.getTime(); + } + // Normalize option cast.number + if(options.cast.number === undefined || options.cast.number === null){ + // Cast number to string using native casting by default + options.cast.number = value => '' + value; + } + // Normalize option cast.object + if(options.cast.object === undefined || options.cast.object === null){ + // Stringify object as JSON by default + options.cast.object = value => JSON.stringify(value); + } + // Normalize option cast.string + if(options.cast.string === undefined || options.cast.string === null){ + // Leave string untouched + options.cast.string = function(value){return value;}; + } + // Normalize option `on_record` + if(options.on_record !== undefined && typeof options.on_record !== 'function'){ + return [Error(`Invalid Option: "on_record" must be a function.`)]; + } + // Normalize option `record_delimiter` + if(options.record_delimiter === undefined || options.record_delimiter === null){ + options.record_delimiter = '\n'; + }else if(isBuffer(options.record_delimiter)){ + options.record_delimiter = options.record_delimiter.toString(); + }else if(typeof options.record_delimiter !== 'string'){ + return [Error(`Invalid Option: record_delimiter must be a buffer or a string, got ${JSON.stringify(options.record_delimiter)}`)]; + } + switch(options.record_delimiter){ + case 'auto': + options.record_delimiter = null; + break; + case 'unix': + options.record_delimiter = "\n"; + break; + case 'mac': + options.record_delimiter = "\r"; + break; + case 'windows': + options.record_delimiter = "\r\n"; + break; + case 'ascii': + options.record_delimiter = "\u001e"; + break; + case 'unicode': + options.record_delimiter = "\u2028"; + break; + } + return [undefined, options]; + }; + + const bom_utf8 = Buffer.from([239, 187, 191]); + + const stringifier = function(options, state, info){ + return { + options: options, + state: state, + info: info, + __transform: function(chunk, push){ + // Chunk validation + if(!Array.isArray(chunk) && typeof chunk !== 'object'){ + return Error(`Invalid Record: expect an array or an object, got ${JSON.stringify(chunk)}`); + } + // Detect columns from the first record + if(this.info.records === 0){ + if(Array.isArray(chunk)){ + if(this.options.header === true && this.options.columns === undefined){ + return Error('Undiscoverable Columns: header option requires column option or object records'); + } + }else if(this.options.columns === undefined){ + const [err, columns] = normalize_columns(Object.keys(chunk)); + if(err) return; + this.options.columns = columns; + } + } + // Emit the header + if(this.info.records === 0){ + this.bom(push); + const err = this.headers(push); + if(err) return err; + } + // Emit and stringify the record if an object or an array + try{ + // this.emit('record', chunk, this.info.records); + if(this.options.on_record){ + this.options.on_record(chunk, this.info.records); + } + }catch(err){ + return err; + } + // Convert the record into a string + let err, chunk_string; + if(this.options.eof){ + [err, chunk_string] = this.stringify(chunk); + if(err) return err; + if(chunk_string === undefined){ + return; + }else { + chunk_string = chunk_string + this.options.record_delimiter; + } + }else { + [err, chunk_string] = this.stringify(chunk); + if(err) return err; + if(chunk_string === undefined){ + return; + }else { + if(this.options.header || this.info.records){ + chunk_string = this.options.record_delimiter + chunk_string; + } + } + } + // Emit the csv + this.info.records++; + push(chunk_string); + }, + stringify: function(chunk, chunkIsHeader=false){ + if(typeof chunk !== 'object'){ + return [undefined, chunk]; + } + const {columns} = this.options; + const record = []; + // Record is an array + if(Array.isArray(chunk)){ + // We are getting an array but the user has specified output columns. In + // this case, we respect the columns indexes + if(columns){ + chunk.splice(columns.length); + } + // Cast record elements + for(let i=0; i= 0; + const containsQuote = (quote !== '') && value.indexOf(quote) >= 0; + const containsEscape = value.indexOf(escape) >= 0 && (escape !== quote); + const containsRecordDelimiter = value.indexOf(record_delimiter) >= 0; + const quotedString = quoted_string && typeof field === 'string'; + let quotedMatch = quoted_match && quoted_match.filter(quoted_match => { + if(typeof quoted_match === 'string'){ + return value.indexOf(quoted_match) !== -1; + }else { + return quoted_match.test(value); + } + }); + quotedMatch = quotedMatch && quotedMatch.length > 0; + const shouldQuote = containsQuote === true || containsdelimiter || containsRecordDelimiter || quoted || quotedString || quotedMatch; + if(shouldQuote === true && containsEscape === true){ + const regexp = escape === '\\' + ? new RegExp(escape + escape, 'g') + : new RegExp(escape, 'g'); + value = value.replace(regexp, escape + escape); + } + if(containsQuote === true){ + const regexp = new RegExp(quote,'g'); + value = value.replace(regexp, escape + quote); + } + if(shouldQuote === true){ + value = quote + value + quote; + } + csvrecord += value; + }else if(quoted_empty === true || (field === '' && quoted_string === true && quoted_empty !== false)){ + csvrecord += quote + quote; + } + if(i !== record.length - 1){ + csvrecord += delimiter; + } + } + return [undefined, csvrecord]; + }, + bom: function(push){ + if(this.options.bom !== true){ + return; + } + push(bom_utf8); + }, + headers: function(push){ + if(this.options.header === false){ + return; + } + if(this.options.columns === undefined){ + return; + } + let err; + let headers = this.options.columns.map(column => column.header); + if(this.options.eof){ + [err, headers] = this.stringify(headers, true); + headers += this.options.record_delimiter; + }else { + [err, headers] = this.stringify(headers); + } + if(err) return err; + push(headers); + }, + __cast: function(value, context){ + const type = typeof value; + try{ + if(type === 'string'){ // Fine for 99% of the cases + return [undefined, this.options.cast.string(value, context)]; + }else if(type === 'bigint'){ + return [undefined, this.options.cast.bigint(value, context)]; + }else if(type === 'number'){ + return [undefined, this.options.cast.number(value, context)]; + }else if(type === 'boolean'){ + return [undefined, this.options.cast.boolean(value, context)]; + }else if(value instanceof Date){ + return [undefined, this.options.cast.date(value, context)]; + }else if(type === 'object' && value !== null){ + return [undefined, this.options.cast.object(value, context)]; + }else { + return [undefined, value, value]; + } + }catch(err){ + return [err]; + } + } + }; + }; + + class Stringifier extends Transform { + constructor(opts = {}){ + super({...{writableObjectMode: true}, ...opts}); + const [err, options] = normalize_options(opts); + if(err !== undefined) throw err; + // Expose options + this.options = options; + // Internal state + this.state = { + stop: false + }; + // Information + this.info = { + records: 0 + }; + this.api = stringifier(this.options, this.state, this.info); + this.api.options.on_record = (...args) => { + this.emit('record', ...args); + }; + } + _transform(chunk, encoding, callback){ + if(this.state.stop === true){ + return; + } + const err = this.api.__transform(chunk, this.push.bind(this)); + if(err !== undefined){ + this.state.stop = true; + } + callback(err); + } + _flush(callback){ + if(this.state.stop === true){ + // Note, Node.js 12 call flush even after an error, we must prevent + // `callback` from being called in flush without any error. + return; + } + if(this.info.records === 0){ + this.api.bom(this.push.bind(this)); + const err = this.api.headers(this.push.bind(this)); + if(err) callback(err); + } + callback(); + } + } + + const stringify = function(){ + let data, options, callback; + for(const i in arguments){ + const argument = arguments[i]; + const type = typeof argument; + if(data === undefined && (Array.isArray(argument))){ + data = argument; + }else if(options === undefined && is_object(argument)){ + options = argument; + }else if(callback === undefined && type === 'function'){ + callback = argument; + }else { + throw new CsvError('CSV_INVALID_ARGUMENT', [ + 'Invalid argument:', + `got ${JSON.stringify(argument)} at index ${i}` + ]); + } + } + const stringifier = new Stringifier(options); + if(callback){ + const chunks = []; + stringifier.on('readable', function(){ + let chunk; + while((chunk = this.read()) !== null){ + chunks.push(chunk); + } + }); + stringifier.on('error', function(err){ + callback(err); + }); + stringifier.on('end', function(){ + callback(undefined, chunks.join('')); + }); + } + if(data !== undefined){ + const writer = function(){ + for(const record of data){ + stringifier.write(record); + } + stringifier.end(); + }; + // Support Deno, Rollup doesnt provide a shim for setImmediate + if(typeof setImmediate === 'function'){ + setImmediate(writer); + }else { + setTimeout(writer, 0); + } + } + return stringifier; + }; + + exports.CsvError = CsvError; + exports.Stringifier = Stringifier; + exports.stringify = stringify; + + Object.defineProperty(exports, '__esModule', { value: true }); + + return exports; })({}); diff --git a/packages/csv-stringify/dist/iife/sync.js b/packages/csv-stringify/dist/iife/sync.js index 452aa65b7..ac4afe5d6 100644 --- a/packages/csv-stringify/dist/iife/sync.js +++ b/packages/csv-stringify/dist/iife/sync.js @@ -202,7 +202,7 @@ var csv_stringify_sync = (function (exports) { var toString = {}.toString; - var isArray$1 = Array.isArray || function (arr) { + var isArray = Array.isArray || function (arr) { return toString.call(arr) == '[object Array]'; }; @@ -470,7 +470,7 @@ var csv_stringify_sync = (function (exports) { return fromArrayLike(that, obj) } - if (obj.type === 'Buffer' && isArray$1(obj.data)) { + if (obj.type === 'Buffer' && isArray(obj.data)) { return fromArrayLike(that, obj.data) } } @@ -535,7 +535,7 @@ var csv_stringify_sync = (function (exports) { }; Buffer.concat = function concat (list, length) { - if (!isArray$1(list)) { + if (!isArray(list)) { throw new TypeError('"list" argument must be an Array of Buffers') } @@ -1966,3038 +1966,6 @@ var csv_stringify_sync = (function (exports) { return typeof obj.readFloatLE === 'function' && typeof obj.slice === 'function' && isFastBuffer(obj.slice(0, 0)) } - var domain; - - // This constructor is used to store event handlers. Instantiating this is - // faster than explicitly calling `Object.create(null)` to get a "clean" empty - // object (tested with v8 v4.9). - function EventHandlers() {} - EventHandlers.prototype = Object.create(null); - - function EventEmitter() { - EventEmitter.init.call(this); - } - - // nodejs oddity - // require('events') === require('events').EventEmitter - EventEmitter.EventEmitter = EventEmitter; - - EventEmitter.usingDomains = false; - - EventEmitter.prototype.domain = undefined; - EventEmitter.prototype._events = undefined; - EventEmitter.prototype._maxListeners = undefined; - - // By default EventEmitters will print a warning if more than 10 listeners are - // added to it. This is a useful default which helps finding memory leaks. - EventEmitter.defaultMaxListeners = 10; - - EventEmitter.init = function() { - this.domain = null; - if (EventEmitter.usingDomains) { - // if there is an active domain, then attach to it. - if (domain.active ) ; - } - - if (!this._events || this._events === Object.getPrototypeOf(this)._events) { - this._events = new EventHandlers(); - this._eventsCount = 0; - } - - this._maxListeners = this._maxListeners || undefined; - }; - - // Obviously not all Emitters should be limited to 10. This function allows - // that to be increased. Set to zero for unlimited. - EventEmitter.prototype.setMaxListeners = function setMaxListeners(n) { - if (typeof n !== 'number' || n < 0 || isNaN(n)) - throw new TypeError('"n" argument must be a positive number'); - this._maxListeners = n; - return this; - }; - - function $getMaxListeners(that) { - if (that._maxListeners === undefined) - return EventEmitter.defaultMaxListeners; - return that._maxListeners; - } - - EventEmitter.prototype.getMaxListeners = function getMaxListeners() { - return $getMaxListeners(this); - }; - - // These standalone emit* functions are used to optimize calling of event - // handlers for fast cases because emit() itself often has a variable number of - // arguments and can be deoptimized because of that. These functions always have - // the same number of arguments and thus do not get deoptimized, so the code - // inside them can execute faster. - function emitNone(handler, isFn, self) { - if (isFn) - handler.call(self); - else { - var len = handler.length; - var listeners = arrayClone(handler, len); - for (var i = 0; i < len; ++i) - listeners[i].call(self); - } - } - function emitOne(handler, isFn, self, arg1) { - if (isFn) - handler.call(self, arg1); - else { - var len = handler.length; - var listeners = arrayClone(handler, len); - for (var i = 0; i < len; ++i) - listeners[i].call(self, arg1); - } - } - function emitTwo(handler, isFn, self, arg1, arg2) { - if (isFn) - handler.call(self, arg1, arg2); - else { - var len = handler.length; - var listeners = arrayClone(handler, len); - for (var i = 0; i < len; ++i) - listeners[i].call(self, arg1, arg2); - } - } - function emitThree(handler, isFn, self, arg1, arg2, arg3) { - if (isFn) - handler.call(self, arg1, arg2, arg3); - else { - var len = handler.length; - var listeners = arrayClone(handler, len); - for (var i = 0; i < len; ++i) - listeners[i].call(self, arg1, arg2, arg3); - } - } - - function emitMany(handler, isFn, self, args) { - if (isFn) - handler.apply(self, args); - else { - var len = handler.length; - var listeners = arrayClone(handler, len); - for (var i = 0; i < len; ++i) - listeners[i].apply(self, args); - } - } - - EventEmitter.prototype.emit = function emit(type) { - var er, handler, len, args, i, events, domain; - var doError = (type === 'error'); - - events = this._events; - if (events) - doError = (doError && events.error == null); - else if (!doError) - return false; - - domain = this.domain; - - // If there is no 'error' event listener then throw. - if (doError) { - er = arguments[1]; - if (domain) { - if (!er) - er = new Error('Uncaught, unspecified "error" event'); - er.domainEmitter = this; - er.domain = domain; - er.domainThrown = false; - domain.emit('error', er); - } else if (er instanceof Error) { - throw er; // Unhandled 'error' event - } else { - // At least give some kind of context to the user - var err = new Error('Uncaught, unspecified "error" event. (' + er + ')'); - err.context = er; - throw err; - } - return false; - } - - handler = events[type]; - - if (!handler) - return false; - - var isFn = typeof handler === 'function'; - len = arguments.length; - switch (len) { - // fast cases - case 1: - emitNone(handler, isFn, this); - break; - case 2: - emitOne(handler, isFn, this, arguments[1]); - break; - case 3: - emitTwo(handler, isFn, this, arguments[1], arguments[2]); - break; - case 4: - emitThree(handler, isFn, this, arguments[1], arguments[2], arguments[3]); - break; - // slower - default: - args = new Array(len - 1); - for (i = 1; i < len; i++) - args[i - 1] = arguments[i]; - emitMany(handler, isFn, this, args); - } - - return true; - }; - - function _addListener(target, type, listener, prepend) { - var m; - var events; - var existing; - - if (typeof listener !== 'function') - throw new TypeError('"listener" argument must be a function'); - - events = target._events; - if (!events) { - events = target._events = new EventHandlers(); - target._eventsCount = 0; - } else { - // To avoid recursion in the case that type === "newListener"! Before - // adding it to the listeners, first emit "newListener". - if (events.newListener) { - target.emit('newListener', type, - listener.listener ? listener.listener : listener); - - // Re-assign `events` because a newListener handler could have caused the - // this._events to be assigned to a new object - events = target._events; - } - existing = events[type]; - } - - if (!existing) { - // Optimize the case of one listener. Don't need the extra array object. - existing = events[type] = listener; - ++target._eventsCount; - } else { - if (typeof existing === 'function') { - // Adding the second element, need to change to array. - existing = events[type] = prepend ? [listener, existing] : - [existing, listener]; - } else { - // If we've already got an array, just append. - if (prepend) { - existing.unshift(listener); - } else { - existing.push(listener); - } - } - - // Check for listener leak - if (!existing.warned) { - m = $getMaxListeners(target); - if (m && m > 0 && existing.length > m) { - existing.warned = true; - var w = new Error('Possible EventEmitter memory leak detected. ' + - existing.length + ' ' + type + ' listeners added. ' + - 'Use emitter.setMaxListeners() to increase limit'); - w.name = 'MaxListenersExceededWarning'; - w.emitter = target; - w.type = type; - w.count = existing.length; - emitWarning(w); - } - } - } - - return target; - } - function emitWarning(e) { - typeof console.warn === 'function' ? console.warn(e) : console.log(e); - } - EventEmitter.prototype.addListener = function addListener(type, listener) { - return _addListener(this, type, listener, false); - }; - - EventEmitter.prototype.on = EventEmitter.prototype.addListener; - - EventEmitter.prototype.prependListener = - function prependListener(type, listener) { - return _addListener(this, type, listener, true); - }; - - function _onceWrap(target, type, listener) { - var fired = false; - function g() { - target.removeListener(type, g); - if (!fired) { - fired = true; - listener.apply(target, arguments); - } - } - g.listener = listener; - return g; - } - - EventEmitter.prototype.once = function once(type, listener) { - if (typeof listener !== 'function') - throw new TypeError('"listener" argument must be a function'); - this.on(type, _onceWrap(this, type, listener)); - return this; - }; - - EventEmitter.prototype.prependOnceListener = - function prependOnceListener(type, listener) { - if (typeof listener !== 'function') - throw new TypeError('"listener" argument must be a function'); - this.prependListener(type, _onceWrap(this, type, listener)); - return this; - }; - - // emits a 'removeListener' event iff the listener was removed - EventEmitter.prototype.removeListener = - function removeListener(type, listener) { - var list, events, position, i, originalListener; - - if (typeof listener !== 'function') - throw new TypeError('"listener" argument must be a function'); - - events = this._events; - if (!events) - return this; - - list = events[type]; - if (!list) - return this; - - if (list === listener || (list.listener && list.listener === listener)) { - if (--this._eventsCount === 0) - this._events = new EventHandlers(); - else { - delete events[type]; - if (events.removeListener) - this.emit('removeListener', type, list.listener || listener); - } - } else if (typeof list !== 'function') { - position = -1; - - for (i = list.length; i-- > 0;) { - if (list[i] === listener || - (list[i].listener && list[i].listener === listener)) { - originalListener = list[i].listener; - position = i; - break; - } - } - - if (position < 0) - return this; - - if (list.length === 1) { - list[0] = undefined; - if (--this._eventsCount === 0) { - this._events = new EventHandlers(); - return this; - } else { - delete events[type]; - } - } else { - spliceOne(list, position); - } - - if (events.removeListener) - this.emit('removeListener', type, originalListener || listener); - } - - return this; - }; - - EventEmitter.prototype.removeAllListeners = - function removeAllListeners(type) { - var listeners, events; - - events = this._events; - if (!events) - return this; - - // not listening for removeListener, no need to emit - if (!events.removeListener) { - if (arguments.length === 0) { - this._events = new EventHandlers(); - this._eventsCount = 0; - } else if (events[type]) { - if (--this._eventsCount === 0) - this._events = new EventHandlers(); - else - delete events[type]; - } - return this; - } - - // emit removeListener for all listeners on all events - if (arguments.length === 0) { - var keys = Object.keys(events); - for (var i = 0, key; i < keys.length; ++i) { - key = keys[i]; - if (key === 'removeListener') continue; - this.removeAllListeners(key); - } - this.removeAllListeners('removeListener'); - this._events = new EventHandlers(); - this._eventsCount = 0; - return this; - } - - listeners = events[type]; - - if (typeof listeners === 'function') { - this.removeListener(type, listeners); - } else if (listeners) { - // LIFO order - do { - this.removeListener(type, listeners[listeners.length - 1]); - } while (listeners[0]); - } - - return this; - }; - - EventEmitter.prototype.listeners = function listeners(type) { - var evlistener; - var ret; - var events = this._events; - - if (!events) - ret = []; - else { - evlistener = events[type]; - if (!evlistener) - ret = []; - else if (typeof evlistener === 'function') - ret = [evlistener.listener || evlistener]; - else - ret = unwrapListeners(evlistener); - } - - return ret; - }; - - EventEmitter.listenerCount = function(emitter, type) { - if (typeof emitter.listenerCount === 'function') { - return emitter.listenerCount(type); - } else { - return listenerCount$1.call(emitter, type); - } - }; - - EventEmitter.prototype.listenerCount = listenerCount$1; - function listenerCount$1(type) { - var events = this._events; - - if (events) { - var evlistener = events[type]; - - if (typeof evlistener === 'function') { - return 1; - } else if (evlistener) { - return evlistener.length; - } - } - - return 0; - } - - EventEmitter.prototype.eventNames = function eventNames() { - return this._eventsCount > 0 ? Reflect.ownKeys(this._events) : []; - }; - - // About 1.5x faster than the two-arg version of Array#splice(). - function spliceOne(list, index) { - for (var i = index, k = i + 1, n = list.length; k < n; i += 1, k += 1) - list[i] = list[k]; - list.pop(); - } - - function arrayClone(arr, i) { - var copy = new Array(i); - while (i--) - copy[i] = arr[i]; - return copy; - } - - function unwrapListeners(arr) { - var ret = new Array(arr.length); - for (var i = 0; i < ret.length; ++i) { - ret[i] = arr[i].listener || arr[i]; - } - return ret; - } - - // shim for using process in browser - // based off https://github.com/defunctzombie/node-process/blob/master/browser.js - - function defaultSetTimout() { - throw new Error('setTimeout has not been defined'); - } - function defaultClearTimeout () { - throw new Error('clearTimeout has not been defined'); - } - var cachedSetTimeout = defaultSetTimout; - var cachedClearTimeout = defaultClearTimeout; - if (typeof global$1.setTimeout === 'function') { - cachedSetTimeout = setTimeout; - } - if (typeof global$1.clearTimeout === 'function') { - cachedClearTimeout = clearTimeout; - } - - function runTimeout(fun) { - if (cachedSetTimeout === setTimeout) { - //normal enviroments in sane situations - return setTimeout(fun, 0); - } - // if setTimeout wasn't available but was latter defined - if ((cachedSetTimeout === defaultSetTimout || !cachedSetTimeout) && setTimeout) { - cachedSetTimeout = setTimeout; - return setTimeout(fun, 0); - } - try { - // when when somebody has screwed with setTimeout but no I.E. maddness - return cachedSetTimeout(fun, 0); - } catch(e){ - try { - // When we are in I.E. but the script has been evaled so I.E. doesn't trust the global object when called normally - return cachedSetTimeout.call(null, fun, 0); - } catch(e){ - // same as above but when it's a version of I.E. that must have the global object for 'this', hopfully our context correct otherwise it will throw a global error - return cachedSetTimeout.call(this, fun, 0); - } - } - - - } - function runClearTimeout(marker) { - if (cachedClearTimeout === clearTimeout) { - //normal enviroments in sane situations - return clearTimeout(marker); - } - // if clearTimeout wasn't available but was latter defined - if ((cachedClearTimeout === defaultClearTimeout || !cachedClearTimeout) && clearTimeout) { - cachedClearTimeout = clearTimeout; - return clearTimeout(marker); - } - try { - // when when somebody has screwed with setTimeout but no I.E. maddness - return cachedClearTimeout(marker); - } catch (e){ - try { - // When we are in I.E. but the script has been evaled so I.E. doesn't trust the global object when called normally - return cachedClearTimeout.call(null, marker); - } catch (e){ - // same as above but when it's a version of I.E. that must have the global object for 'this', hopfully our context correct otherwise it will throw a global error. - // Some versions of I.E. have different rules for clearTimeout vs setTimeout - return cachedClearTimeout.call(this, marker); - } - } - - - - } - var queue = []; - var draining = false; - var currentQueue; - var queueIndex = -1; - - function cleanUpNextTick() { - if (!draining || !currentQueue) { - return; - } - draining = false; - if (currentQueue.length) { - queue = currentQueue.concat(queue); - } else { - queueIndex = -1; - } - if (queue.length) { - drainQueue(); - } - } - - function drainQueue() { - if (draining) { - return; - } - var timeout = runTimeout(cleanUpNextTick); - draining = true; - - var len = queue.length; - while(len) { - currentQueue = queue; - queue = []; - while (++queueIndex < len) { - if (currentQueue) { - currentQueue[queueIndex].run(); - } - } - queueIndex = -1; - len = queue.length; - } - currentQueue = null; - draining = false; - runClearTimeout(timeout); - } - function nextTick(fun) { - var args = new Array(arguments.length - 1); - if (arguments.length > 1) { - for (var i = 1; i < arguments.length; i++) { - args[i - 1] = arguments[i]; - } - } - queue.push(new Item(fun, args)); - if (queue.length === 1 && !draining) { - runTimeout(drainQueue); - } - } - // v8 likes predictible objects - function Item(fun, array) { - this.fun = fun; - this.array = array; - } - Item.prototype.run = function () { - this.fun.apply(null, this.array); - }; - - // from https://github.com/kumavis/browser-process-hrtime/blob/master/index.js - var performance = global$1.performance || {}; - performance.now || - performance.mozNow || - performance.msNow || - performance.oNow || - performance.webkitNow || - function(){ return (new Date()).getTime() }; - - var inherits; - if (typeof Object.create === 'function'){ - inherits = function inherits(ctor, superCtor) { - // implementation from standard node.js 'util' module - ctor.super_ = superCtor; - ctor.prototype = Object.create(superCtor.prototype, { - constructor: { - value: ctor, - enumerable: false, - writable: true, - configurable: true - } - }); - }; - } else { - inherits = function inherits(ctor, superCtor) { - ctor.super_ = superCtor; - var TempCtor = function () {}; - TempCtor.prototype = superCtor.prototype; - ctor.prototype = new TempCtor(); - ctor.prototype.constructor = ctor; - }; - } - var inherits$1 = inherits; - - var formatRegExp = /%[sdj%]/g; - function format(f) { - if (!isString(f)) { - var objects = []; - for (var i = 0; i < arguments.length; i++) { - objects.push(inspect(arguments[i])); - } - return objects.join(' '); - } - - var i = 1; - var args = arguments; - var len = args.length; - var str = String(f).replace(formatRegExp, function(x) { - if (x === '%%') return '%'; - if (i >= len) return x; - switch (x) { - case '%s': return String(args[i++]); - case '%d': return Number(args[i++]); - case '%j': - try { - return JSON.stringify(args[i++]); - } catch (_) { - return '[Circular]'; - } - default: - return x; - } - }); - for (var x = args[i]; i < len; x = args[++i]) { - if (isNull(x) || !isObject$1(x)) { - str += ' ' + x; - } else { - str += ' ' + inspect(x); - } - } - return str; - } - - // Mark that a method should not be used. - // Returns a modified function which warns once by default. - // If --no-deprecation is set, then it is a no-op. - function deprecate(fn, msg) { - // Allow for deprecating things in the process of starting up. - if (isUndefined(global$1.process)) { - return function() { - return deprecate(fn, msg).apply(this, arguments); - }; - } - - var warned = false; - function deprecated() { - if (!warned) { - { - console.error(msg); - } - warned = true; - } - return fn.apply(this, arguments); - } - - return deprecated; - } - - var debugs = {}; - var debugEnviron; - function debuglog(set) { - if (isUndefined(debugEnviron)) - debugEnviron = ''; - set = set.toUpperCase(); - if (!debugs[set]) { - if (new RegExp('\\b' + set + '\\b', 'i').test(debugEnviron)) { - var pid = 0; - debugs[set] = function() { - var msg = format.apply(null, arguments); - console.error('%s %d: %s', set, pid, msg); - }; - } else { - debugs[set] = function() {}; - } - } - return debugs[set]; - } - - /** - * Echos the value of a value. Trys to print the value out - * in the best way possible given the different types. - * - * @param {Object} obj The object to print out. - * @param {Object} opts Optional options object that alters the output. - */ - /* legacy: obj, showHidden, depth, colors*/ - function inspect(obj, opts) { - // default options - var ctx = { - seen: [], - stylize: stylizeNoColor - }; - // legacy... - if (arguments.length >= 3) ctx.depth = arguments[2]; - if (arguments.length >= 4) ctx.colors = arguments[3]; - if (isBoolean(opts)) { - // legacy... - ctx.showHidden = opts; - } else if (opts) { - // got an "options" object - _extend(ctx, opts); - } - // set default options - if (isUndefined(ctx.showHidden)) ctx.showHidden = false; - if (isUndefined(ctx.depth)) ctx.depth = 2; - if (isUndefined(ctx.colors)) ctx.colors = false; - if (isUndefined(ctx.customInspect)) ctx.customInspect = true; - if (ctx.colors) ctx.stylize = stylizeWithColor; - return formatValue(ctx, obj, ctx.depth); - } - - // http://en.wikipedia.org/wiki/ANSI_escape_code#graphics - inspect.colors = { - 'bold' : [1, 22], - 'italic' : [3, 23], - 'underline' : [4, 24], - 'inverse' : [7, 27], - 'white' : [37, 39], - 'grey' : [90, 39], - 'black' : [30, 39], - 'blue' : [34, 39], - 'cyan' : [36, 39], - 'green' : [32, 39], - 'magenta' : [35, 39], - 'red' : [31, 39], - 'yellow' : [33, 39] - }; - - // Don't use 'blue' not visible on cmd.exe - inspect.styles = { - 'special': 'cyan', - 'number': 'yellow', - 'boolean': 'yellow', - 'undefined': 'grey', - 'null': 'bold', - 'string': 'green', - 'date': 'magenta', - // "name": intentionally not styling - 'regexp': 'red' - }; - - - function stylizeWithColor(str, styleType) { - var style = inspect.styles[styleType]; - - if (style) { - return '\u001b[' + inspect.colors[style][0] + 'm' + str + - '\u001b[' + inspect.colors[style][1] + 'm'; - } else { - return str; - } - } - - - function stylizeNoColor(str, styleType) { - return str; - } - - - function arrayToHash(array) { - var hash = {}; - - array.forEach(function(val, idx) { - hash[val] = true; - }); - - return hash; - } - - - function formatValue(ctx, value, recurseTimes) { - // Provide a hook for user-specified inspect functions. - // Check that value is an object with an inspect function on it - if (ctx.customInspect && - value && - isFunction(value.inspect) && - // Filter out the util module, it's inspect function is special - value.inspect !== inspect && - // Also filter out any prototype objects using the circular check. - !(value.constructor && value.constructor.prototype === value)) { - var ret = value.inspect(recurseTimes, ctx); - if (!isString(ret)) { - ret = formatValue(ctx, ret, recurseTimes); - } - return ret; - } - - // Primitive types cannot have properties - var primitive = formatPrimitive(ctx, value); - if (primitive) { - return primitive; - } - - // Look up the keys of the object. - var keys = Object.keys(value); - var visibleKeys = arrayToHash(keys); - - if (ctx.showHidden) { - keys = Object.getOwnPropertyNames(value); - } - - // IE doesn't make error fields non-enumerable - // http://msdn.microsoft.com/en-us/library/ie/dww52sbt(v=vs.94).aspx - if (isError(value) - && (keys.indexOf('message') >= 0 || keys.indexOf('description') >= 0)) { - return formatError(value); - } - - // Some type of object without properties can be shortcutted. - if (keys.length === 0) { - if (isFunction(value)) { - var name = value.name ? ': ' + value.name : ''; - return ctx.stylize('[Function' + name + ']', 'special'); - } - if (isRegExp(value)) { - return ctx.stylize(RegExp.prototype.toString.call(value), 'regexp'); - } - if (isDate(value)) { - return ctx.stylize(Date.prototype.toString.call(value), 'date'); - } - if (isError(value)) { - return formatError(value); - } - } - - var base = '', array = false, braces = ['{', '}']; - - // Make Array say that they are Array - if (isArray(value)) { - array = true; - braces = ['[', ']']; - } - - // Make functions say that they are functions - if (isFunction(value)) { - var n = value.name ? ': ' + value.name : ''; - base = ' [Function' + n + ']'; - } - - // Make RegExps say that they are RegExps - if (isRegExp(value)) { - base = ' ' + RegExp.prototype.toString.call(value); - } - - // Make dates with properties first say the date - if (isDate(value)) { - base = ' ' + Date.prototype.toUTCString.call(value); - } - - // Make error with message first say the error - if (isError(value)) { - base = ' ' + formatError(value); - } - - if (keys.length === 0 && (!array || value.length == 0)) { - return braces[0] + base + braces[1]; - } - - if (recurseTimes < 0) { - if (isRegExp(value)) { - return ctx.stylize(RegExp.prototype.toString.call(value), 'regexp'); - } else { - return ctx.stylize('[Object]', 'special'); - } - } - - ctx.seen.push(value); - - var output; - if (array) { - output = formatArray(ctx, value, recurseTimes, visibleKeys, keys); - } else { - output = keys.map(function(key) { - return formatProperty(ctx, value, recurseTimes, visibleKeys, key, array); - }); - } - - ctx.seen.pop(); - - return reduceToSingleString(output, base, braces); - } - - - function formatPrimitive(ctx, value) { - if (isUndefined(value)) - return ctx.stylize('undefined', 'undefined'); - if (isString(value)) { - var simple = '\'' + JSON.stringify(value).replace(/^"|"$/g, '') - .replace(/'/g, "\\'") - .replace(/\\"/g, '"') + '\''; - return ctx.stylize(simple, 'string'); - } - if (isNumber(value)) - return ctx.stylize('' + value, 'number'); - if (isBoolean(value)) - return ctx.stylize('' + value, 'boolean'); - // For some reason typeof null is "object", so special case here. - if (isNull(value)) - return ctx.stylize('null', 'null'); - } - - - function formatError(value) { - return '[' + Error.prototype.toString.call(value) + ']'; - } - - - function formatArray(ctx, value, recurseTimes, visibleKeys, keys) { - var output = []; - for (var i = 0, l = value.length; i < l; ++i) { - if (hasOwnProperty(value, String(i))) { - output.push(formatProperty(ctx, value, recurseTimes, visibleKeys, - String(i), true)); - } else { - output.push(''); - } - } - keys.forEach(function(key) { - if (!key.match(/^\d+$/)) { - output.push(formatProperty(ctx, value, recurseTimes, visibleKeys, - key, true)); - } - }); - return output; - } - - - function formatProperty(ctx, value, recurseTimes, visibleKeys, key, array) { - var name, str, desc; - desc = Object.getOwnPropertyDescriptor(value, key) || { value: value[key] }; - if (desc.get) { - if (desc.set) { - str = ctx.stylize('[Getter/Setter]', 'special'); - } else { - str = ctx.stylize('[Getter]', 'special'); - } - } else { - if (desc.set) { - str = ctx.stylize('[Setter]', 'special'); - } - } - if (!hasOwnProperty(visibleKeys, key)) { - name = '[' + key + ']'; - } - if (!str) { - if (ctx.seen.indexOf(desc.value) < 0) { - if (isNull(recurseTimes)) { - str = formatValue(ctx, desc.value, null); - } else { - str = formatValue(ctx, desc.value, recurseTimes - 1); - } - if (str.indexOf('\n') > -1) { - if (array) { - str = str.split('\n').map(function(line) { - return ' ' + line; - }).join('\n').substr(2); - } else { - str = '\n' + str.split('\n').map(function(line) { - return ' ' + line; - }).join('\n'); - } - } - } else { - str = ctx.stylize('[Circular]', 'special'); - } - } - if (isUndefined(name)) { - if (array && key.match(/^\d+$/)) { - return str; - } - name = JSON.stringify('' + key); - if (name.match(/^"([a-zA-Z_][a-zA-Z_0-9]*)"$/)) { - name = name.substr(1, name.length - 2); - name = ctx.stylize(name, 'name'); - } else { - name = name.replace(/'/g, "\\'") - .replace(/\\"/g, '"') - .replace(/(^"|"$)/g, "'"); - name = ctx.stylize(name, 'string'); - } - } - - return name + ': ' + str; - } - - - function reduceToSingleString(output, base, braces) { - var length = output.reduce(function(prev, cur) { - if (cur.indexOf('\n') >= 0) ; - return prev + cur.replace(/\u001b\[\d\d?m/g, '').length + 1; - }, 0); - - if (length > 60) { - return braces[0] + - (base === '' ? '' : base + '\n ') + - ' ' + - output.join(',\n ') + - ' ' + - braces[1]; - } - - return braces[0] + base + ' ' + output.join(', ') + ' ' + braces[1]; - } - - - // NOTE: These type checking functions intentionally don't use `instanceof` - // because it is fragile and can be easily faked with `Object.create()`. - function isArray(ar) { - return Array.isArray(ar); - } - - function isBoolean(arg) { - return typeof arg === 'boolean'; - } - - function isNull(arg) { - return arg === null; - } - - function isNumber(arg) { - return typeof arg === 'number'; - } - - function isString(arg) { - return typeof arg === 'string'; - } - - function isUndefined(arg) { - return arg === void 0; - } - - function isRegExp(re) { - return isObject$1(re) && objectToString(re) === '[object RegExp]'; - } - - function isObject$1(arg) { - return typeof arg === 'object' && arg !== null; - } - - function isDate(d) { - return isObject$1(d) && objectToString(d) === '[object Date]'; - } - - function isError(e) { - return isObject$1(e) && - (objectToString(e) === '[object Error]' || e instanceof Error); - } - - function isFunction(arg) { - return typeof arg === 'function'; - } - - function objectToString(o) { - return Object.prototype.toString.call(o); - } - - function _extend(origin, add) { - // Don't do anything if add isn't an object - if (!add || !isObject$1(add)) return origin; - - var keys = Object.keys(add); - var i = keys.length; - while (i--) { - origin[keys[i]] = add[keys[i]]; - } - return origin; - } - function hasOwnProperty(obj, prop) { - return Object.prototype.hasOwnProperty.call(obj, prop); - } - - function BufferList() { - this.head = null; - this.tail = null; - this.length = 0; - } - - BufferList.prototype.push = function (v) { - var entry = { data: v, next: null }; - if (this.length > 0) this.tail.next = entry;else this.head = entry; - this.tail = entry; - ++this.length; - }; - - BufferList.prototype.unshift = function (v) { - var entry = { data: v, next: this.head }; - if (this.length === 0) this.tail = entry; - this.head = entry; - ++this.length; - }; - - BufferList.prototype.shift = function () { - if (this.length === 0) return; - var ret = this.head.data; - if (this.length === 1) this.head = this.tail = null;else this.head = this.head.next; - --this.length; - return ret; - }; - - BufferList.prototype.clear = function () { - this.head = this.tail = null; - this.length = 0; - }; - - BufferList.prototype.join = function (s) { - if (this.length === 0) return ''; - var p = this.head; - var ret = '' + p.data; - while (p = p.next) { - ret += s + p.data; - }return ret; - }; - - BufferList.prototype.concat = function (n) { - if (this.length === 0) return Buffer.alloc(0); - if (this.length === 1) return this.head.data; - var ret = Buffer.allocUnsafe(n >>> 0); - var p = this.head; - var i = 0; - while (p) { - p.data.copy(ret, i); - i += p.data.length; - p = p.next; - } - return ret; - }; - - // Copyright Joyent, Inc. and other Node contributors. - var isBufferEncoding = Buffer.isEncoding - || function(encoding) { - switch (encoding && encoding.toLowerCase()) { - case 'hex': case 'utf8': case 'utf-8': case 'ascii': case 'binary': case 'base64': case 'ucs2': case 'ucs-2': case 'utf16le': case 'utf-16le': case 'raw': return true; - default: return false; - } - }; - - - function assertEncoding(encoding) { - if (encoding && !isBufferEncoding(encoding)) { - throw new Error('Unknown encoding: ' + encoding); - } - } - - // StringDecoder provides an interface for efficiently splitting a series of - // buffers into a series of JS strings without breaking apart multi-byte - // characters. CESU-8 is handled as part of the UTF-8 encoding. - // - // @TODO Handling all encodings inside a single object makes it very difficult - // to reason about this code, so it should be split up in the future. - // @TODO There should be a utf8-strict encoding that rejects invalid UTF-8 code - // points as used by CESU-8. - function StringDecoder(encoding) { - this.encoding = (encoding || 'utf8').toLowerCase().replace(/[-_]/, ''); - assertEncoding(encoding); - switch (this.encoding) { - case 'utf8': - // CESU-8 represents each of Surrogate Pair by 3-bytes - this.surrogateSize = 3; - break; - case 'ucs2': - case 'utf16le': - // UTF-16 represents each of Surrogate Pair by 2-bytes - this.surrogateSize = 2; - this.detectIncompleteChar = utf16DetectIncompleteChar; - break; - case 'base64': - // Base-64 stores 3 bytes in 4 chars, and pads the remainder. - this.surrogateSize = 3; - this.detectIncompleteChar = base64DetectIncompleteChar; - break; - default: - this.write = passThroughWrite; - return; - } - - // Enough space to store all bytes of a single character. UTF-8 needs 4 - // bytes, but CESU-8 may require up to 6 (3 bytes per surrogate). - this.charBuffer = new Buffer(6); - // Number of bytes received for the current incomplete multi-byte character. - this.charReceived = 0; - // Number of bytes expected for the current incomplete multi-byte character. - this.charLength = 0; - } - - // write decodes the given buffer and returns it as JS string that is - // guaranteed to not contain any partial multi-byte characters. Any partial - // character found at the end of the buffer is buffered up, and will be - // returned when calling write again with the remaining bytes. - // - // Note: Converting a Buffer containing an orphan surrogate to a String - // currently works, but converting a String to a Buffer (via `new Buffer`, or - // Buffer#write) will replace incomplete surrogates with the unicode - // replacement character. See https://codereview.chromium.org/121173009/ . - StringDecoder.prototype.write = function(buffer) { - var charStr = ''; - // if our last write ended with an incomplete multibyte character - while (this.charLength) { - // determine how many remaining bytes this buffer has to offer for this char - var available = (buffer.length >= this.charLength - this.charReceived) ? - this.charLength - this.charReceived : - buffer.length; - - // add the new bytes to the char buffer - buffer.copy(this.charBuffer, this.charReceived, 0, available); - this.charReceived += available; - - if (this.charReceived < this.charLength) { - // still not enough chars in this buffer? wait for more ... - return ''; - } - - // remove bytes belonging to the current character from the buffer - buffer = buffer.slice(available, buffer.length); - - // get the character that was split - charStr = this.charBuffer.slice(0, this.charLength).toString(this.encoding); - - // CESU-8: lead surrogate (D800-DBFF) is also the incomplete character - var charCode = charStr.charCodeAt(charStr.length - 1); - if (charCode >= 0xD800 && charCode <= 0xDBFF) { - this.charLength += this.surrogateSize; - charStr = ''; - continue; - } - this.charReceived = this.charLength = 0; - - // if there are no more bytes in this buffer, just emit our char - if (buffer.length === 0) { - return charStr; - } - break; - } - - // determine and set charLength / charReceived - this.detectIncompleteChar(buffer); - - var end = buffer.length; - if (this.charLength) { - // buffer the incomplete character bytes we got - buffer.copy(this.charBuffer, 0, buffer.length - this.charReceived, end); - end -= this.charReceived; - } - - charStr += buffer.toString(this.encoding, 0, end); - - var end = charStr.length - 1; - var charCode = charStr.charCodeAt(end); - // CESU-8: lead surrogate (D800-DBFF) is also the incomplete character - if (charCode >= 0xD800 && charCode <= 0xDBFF) { - var size = this.surrogateSize; - this.charLength += size; - this.charReceived += size; - this.charBuffer.copy(this.charBuffer, size, 0, size); - buffer.copy(this.charBuffer, 0, 0, size); - return charStr.substring(0, end); - } - - // or just emit the charStr - return charStr; - }; - - // detectIncompleteChar determines if there is an incomplete UTF-8 character at - // the end of the given buffer. If so, it sets this.charLength to the byte - // length that character, and sets this.charReceived to the number of bytes - // that are available for this character. - StringDecoder.prototype.detectIncompleteChar = function(buffer) { - // determine how many bytes we have to check at the end of this buffer - var i = (buffer.length >= 3) ? 3 : buffer.length; - - // Figure out if one of the last i bytes of our buffer announces an - // incomplete char. - for (; i > 0; i--) { - var c = buffer[buffer.length - i]; - - // See http://en.wikipedia.org/wiki/UTF-8#Description - - // 110XXXXX - if (i == 1 && c >> 5 == 0x06) { - this.charLength = 2; - break; - } - - // 1110XXXX - if (i <= 2 && c >> 4 == 0x0E) { - this.charLength = 3; - break; - } - - // 11110XXX - if (i <= 3 && c >> 3 == 0x1E) { - this.charLength = 4; - break; - } - } - this.charReceived = i; - }; - - StringDecoder.prototype.end = function(buffer) { - var res = ''; - if (buffer && buffer.length) - res = this.write(buffer); - - if (this.charReceived) { - var cr = this.charReceived; - var buf = this.charBuffer; - var enc = this.encoding; - res += buf.slice(0, cr).toString(enc); - } - - return res; - }; - - function passThroughWrite(buffer) { - return buffer.toString(this.encoding); - } - - function utf16DetectIncompleteChar(buffer) { - this.charReceived = buffer.length % 2; - this.charLength = this.charReceived ? 2 : 0; - } - - function base64DetectIncompleteChar(buffer) { - this.charReceived = buffer.length % 3; - this.charLength = this.charReceived ? 3 : 0; - } - - Readable.ReadableState = ReadableState; - - var debug = debuglog('stream'); - inherits$1(Readable, EventEmitter); - - function prependListener(emitter, event, fn) { - // Sadly this is not cacheable as some libraries bundle their own - // event emitter implementation with them. - if (typeof emitter.prependListener === 'function') { - return emitter.prependListener(event, fn); - } else { - // This is a hack to make sure that our error handler is attached before any - // userland ones. NEVER DO THIS. This is here only because this code needs - // to continue to work with older versions of Node.js that do not include - // the prependListener() method. The goal is to eventually remove this hack. - if (!emitter._events || !emitter._events[event]) - emitter.on(event, fn); - else if (Array.isArray(emitter._events[event])) - emitter._events[event].unshift(fn); - else - emitter._events[event] = [fn, emitter._events[event]]; - } - } - function listenerCount (emitter, type) { - return emitter.listeners(type).length; - } - function ReadableState(options, stream) { - - options = options || {}; - - // object stream flag. Used to make read(n) ignore n and to - // make all the buffer merging and length checks go away - this.objectMode = !!options.objectMode; - - if (stream instanceof Duplex) this.objectMode = this.objectMode || !!options.readableObjectMode; - - // the point at which it stops calling _read() to fill the buffer - // Note: 0 is a valid value, means "don't call _read preemptively ever" - var hwm = options.highWaterMark; - var defaultHwm = this.objectMode ? 16 : 16 * 1024; - this.highWaterMark = hwm || hwm === 0 ? hwm : defaultHwm; - - // cast to ints. - this.highWaterMark = ~ ~this.highWaterMark; - - // A linked list is used to store data chunks instead of an array because the - // linked list can remove elements from the beginning faster than - // array.shift() - this.buffer = new BufferList(); - this.length = 0; - this.pipes = null; - this.pipesCount = 0; - this.flowing = null; - this.ended = false; - this.endEmitted = false; - this.reading = false; - - // a flag to be able to tell if the onwrite cb is called immediately, - // or on a later tick. We set this to true at first, because any - // actions that shouldn't happen until "later" should generally also - // not happen before the first write call. - this.sync = true; - - // whenever we return null, then we set a flag to say - // that we're awaiting a 'readable' event emission. - this.needReadable = false; - this.emittedReadable = false; - this.readableListening = false; - this.resumeScheduled = false; - - // Crypto is kind of old and crusty. Historically, its default string - // encoding is 'binary' so we have to make this configurable. - // Everything else in the universe uses 'utf8', though. - this.defaultEncoding = options.defaultEncoding || 'utf8'; - - // when piping, we only care about 'readable' events that happen - // after read()ing all the bytes and not getting any pushback. - this.ranOut = false; - - // the number of writers that are awaiting a drain event in .pipe()s - this.awaitDrain = 0; - - // if true, a maybeReadMore has been scheduled - this.readingMore = false; - - this.decoder = null; - this.encoding = null; - if (options.encoding) { - this.decoder = new StringDecoder(options.encoding); - this.encoding = options.encoding; - } - } - function Readable(options) { - - if (!(this instanceof Readable)) return new Readable(options); - - this._readableState = new ReadableState(options, this); - - // legacy - this.readable = true; - - if (options && typeof options.read === 'function') this._read = options.read; - - EventEmitter.call(this); - } - - // Manually shove something into the read() buffer. - // This returns true if the highWaterMark has not been hit yet, - // similar to how Writable.write() returns true if you should - // write() some more. - Readable.prototype.push = function (chunk, encoding) { - var state = this._readableState; - - if (!state.objectMode && typeof chunk === 'string') { - encoding = encoding || state.defaultEncoding; - if (encoding !== state.encoding) { - chunk = Buffer.from(chunk, encoding); - encoding = ''; - } - } - - return readableAddChunk(this, state, chunk, encoding, false); - }; - - // Unshift should *always* be something directly out of read() - Readable.prototype.unshift = function (chunk) { - var state = this._readableState; - return readableAddChunk(this, state, chunk, '', true); - }; - - Readable.prototype.isPaused = function () { - return this._readableState.flowing === false; - }; - - function readableAddChunk(stream, state, chunk, encoding, addToFront) { - var er = chunkInvalid(state, chunk); - if (er) { - stream.emit('error', er); - } else if (chunk === null) { - state.reading = false; - onEofChunk(stream, state); - } else if (state.objectMode || chunk && chunk.length > 0) { - if (state.ended && !addToFront) { - var e = new Error('stream.push() after EOF'); - stream.emit('error', e); - } else if (state.endEmitted && addToFront) { - var _e = new Error('stream.unshift() after end event'); - stream.emit('error', _e); - } else { - var skipAdd; - if (state.decoder && !addToFront && !encoding) { - chunk = state.decoder.write(chunk); - skipAdd = !state.objectMode && chunk.length === 0; - } - - if (!addToFront) state.reading = false; - - // Don't add to the buffer if we've decoded to an empty string chunk and - // we're not in object mode - if (!skipAdd) { - // if we want the data now, just emit it. - if (state.flowing && state.length === 0 && !state.sync) { - stream.emit('data', chunk); - stream.read(0); - } else { - // update the buffer info. - state.length += state.objectMode ? 1 : chunk.length; - if (addToFront) state.buffer.unshift(chunk);else state.buffer.push(chunk); - - if (state.needReadable) emitReadable(stream); - } - } - - maybeReadMore(stream, state); - } - } else if (!addToFront) { - state.reading = false; - } - - return needMoreData(state); - } - - // if it's past the high water mark, we can push in some more. - // Also, if we have no data yet, we can stand some - // more bytes. This is to work around cases where hwm=0, - // such as the repl. Also, if the push() triggered a - // readable event, and the user called read(largeNumber) such that - // needReadable was set, then we ought to push more, so that another - // 'readable' event will be triggered. - function needMoreData(state) { - return !state.ended && (state.needReadable || state.length < state.highWaterMark || state.length === 0); - } - - // backwards compatibility. - Readable.prototype.setEncoding = function (enc) { - this._readableState.decoder = new StringDecoder(enc); - this._readableState.encoding = enc; - return this; - }; - - // Don't raise the hwm > 8MB - var MAX_HWM = 0x800000; - function computeNewHighWaterMark(n) { - if (n >= MAX_HWM) { - n = MAX_HWM; - } else { - // Get the next highest power of 2 to prevent increasing hwm excessively in - // tiny amounts - n--; - n |= n >>> 1; - n |= n >>> 2; - n |= n >>> 4; - n |= n >>> 8; - n |= n >>> 16; - n++; - } - return n; - } - - // This function is designed to be inlinable, so please take care when making - // changes to the function body. - function howMuchToRead(n, state) { - if (n <= 0 || state.length === 0 && state.ended) return 0; - if (state.objectMode) return 1; - if (n !== n) { - // Only flow one buffer at a time - if (state.flowing && state.length) return state.buffer.head.data.length;else return state.length; - } - // If we're asking for more than the current hwm, then raise the hwm. - if (n > state.highWaterMark) state.highWaterMark = computeNewHighWaterMark(n); - if (n <= state.length) return n; - // Don't have enough - if (!state.ended) { - state.needReadable = true; - return 0; - } - return state.length; - } - - // you can override either this method, or the async _read(n) below. - Readable.prototype.read = function (n) { - debug('read', n); - n = parseInt(n, 10); - var state = this._readableState; - var nOrig = n; - - if (n !== 0) state.emittedReadable = false; - - // if we're doing read(0) to trigger a readable event, but we - // already have a bunch of data in the buffer, then just trigger - // the 'readable' event and move on. - if (n === 0 && state.needReadable && (state.length >= state.highWaterMark || state.ended)) { - debug('read: emitReadable', state.length, state.ended); - if (state.length === 0 && state.ended) endReadable(this);else emitReadable(this); - return null; - } - - n = howMuchToRead(n, state); - - // if we've ended, and we're now clear, then finish it up. - if (n === 0 && state.ended) { - if (state.length === 0) endReadable(this); - return null; - } - - // All the actual chunk generation logic needs to be - // *below* the call to _read. The reason is that in certain - // synthetic stream cases, such as passthrough streams, _read - // may be a completely synchronous operation which may change - // the state of the read buffer, providing enough data when - // before there was *not* enough. - // - // So, the steps are: - // 1. Figure out what the state of things will be after we do - // a read from the buffer. - // - // 2. If that resulting state will trigger a _read, then call _read. - // Note that this may be asynchronous, or synchronous. Yes, it is - // deeply ugly to write APIs this way, but that still doesn't mean - // that the Readable class should behave improperly, as streams are - // designed to be sync/async agnostic. - // Take note if the _read call is sync or async (ie, if the read call - // has returned yet), so that we know whether or not it's safe to emit - // 'readable' etc. - // - // 3. Actually pull the requested chunks out of the buffer and return. - - // if we need a readable event, then we need to do some reading. - var doRead = state.needReadable; - debug('need readable', doRead); - - // if we currently have less than the highWaterMark, then also read some - if (state.length === 0 || state.length - n < state.highWaterMark) { - doRead = true; - debug('length less than watermark', doRead); - } - - // however, if we've ended, then there's no point, and if we're already - // reading, then it's unnecessary. - if (state.ended || state.reading) { - doRead = false; - debug('reading or ended', doRead); - } else if (doRead) { - debug('do read'); - state.reading = true; - state.sync = true; - // if the length is currently zero, then we *need* a readable event. - if (state.length === 0) state.needReadable = true; - // call internal read method - this._read(state.highWaterMark); - state.sync = false; - // If _read pushed data synchronously, then `reading` will be false, - // and we need to re-evaluate how much data we can return to the user. - if (!state.reading) n = howMuchToRead(nOrig, state); - } - - var ret; - if (n > 0) ret = fromList(n, state);else ret = null; - - if (ret === null) { - state.needReadable = true; - n = 0; - } else { - state.length -= n; - } - - if (state.length === 0) { - // If we have nothing in the buffer, then we want to know - // as soon as we *do* get something into the buffer. - if (!state.ended) state.needReadable = true; - - // If we tried to read() past the EOF, then emit end on the next tick. - if (nOrig !== n && state.ended) endReadable(this); - } - - if (ret !== null) this.emit('data', ret); - - return ret; - }; - - function chunkInvalid(state, chunk) { - var er = null; - if (!isBuffer(chunk) && typeof chunk !== 'string' && chunk !== null && chunk !== undefined && !state.objectMode) { - er = new TypeError('Invalid non-string/buffer chunk'); - } - return er; - } - - function onEofChunk(stream, state) { - if (state.ended) return; - if (state.decoder) { - var chunk = state.decoder.end(); - if (chunk && chunk.length) { - state.buffer.push(chunk); - state.length += state.objectMode ? 1 : chunk.length; - } - } - state.ended = true; - - // emit 'readable' now to make sure it gets picked up. - emitReadable(stream); - } - - // Don't emit readable right away in sync mode, because this can trigger - // another read() call => stack overflow. This way, it might trigger - // a nextTick recursion warning, but that's not so bad. - function emitReadable(stream) { - var state = stream._readableState; - state.needReadable = false; - if (!state.emittedReadable) { - debug('emitReadable', state.flowing); - state.emittedReadable = true; - if (state.sync) nextTick(emitReadable_, stream);else emitReadable_(stream); - } - } - - function emitReadable_(stream) { - debug('emit readable'); - stream.emit('readable'); - flow(stream); - } - - // at this point, the user has presumably seen the 'readable' event, - // and called read() to consume some data. that may have triggered - // in turn another _read(n) call, in which case reading = true if - // it's in progress. - // However, if we're not ended, or reading, and the length < hwm, - // then go ahead and try to read some more preemptively. - function maybeReadMore(stream, state) { - if (!state.readingMore) { - state.readingMore = true; - nextTick(maybeReadMore_, stream, state); - } - } - - function maybeReadMore_(stream, state) { - var len = state.length; - while (!state.reading && !state.flowing && !state.ended && state.length < state.highWaterMark) { - debug('maybeReadMore read 0'); - stream.read(0); - if (len === state.length) - // didn't get any data, stop spinning. - break;else len = state.length; - } - state.readingMore = false; - } - - // abstract method. to be overridden in specific implementation classes. - // call cb(er, data) where data is <= n in length. - // for virtual (non-string, non-buffer) streams, "length" is somewhat - // arbitrary, and perhaps not very meaningful. - Readable.prototype._read = function (n) { - this.emit('error', new Error('not implemented')); - }; - - Readable.prototype.pipe = function (dest, pipeOpts) { - var src = this; - var state = this._readableState; - - switch (state.pipesCount) { - case 0: - state.pipes = dest; - break; - case 1: - state.pipes = [state.pipes, dest]; - break; - default: - state.pipes.push(dest); - break; - } - state.pipesCount += 1; - debug('pipe count=%d opts=%j', state.pipesCount, pipeOpts); - - var doEnd = (!pipeOpts || pipeOpts.end !== false); - - var endFn = doEnd ? onend : cleanup; - if (state.endEmitted) nextTick(endFn);else src.once('end', endFn); - - dest.on('unpipe', onunpipe); - function onunpipe(readable) { - debug('onunpipe'); - if (readable === src) { - cleanup(); - } - } - - function onend() { - debug('onend'); - dest.end(); - } - - // when the dest drains, it reduces the awaitDrain counter - // on the source. This would be more elegant with a .once() - // handler in flow(), but adding and removing repeatedly is - // too slow. - var ondrain = pipeOnDrain(src); - dest.on('drain', ondrain); - - var cleanedUp = false; - function cleanup() { - debug('cleanup'); - // cleanup event handlers once the pipe is broken - dest.removeListener('close', onclose); - dest.removeListener('finish', onfinish); - dest.removeListener('drain', ondrain); - dest.removeListener('error', onerror); - dest.removeListener('unpipe', onunpipe); - src.removeListener('end', onend); - src.removeListener('end', cleanup); - src.removeListener('data', ondata); - - cleanedUp = true; - - // if the reader is waiting for a drain event from this - // specific writer, then it would cause it to never start - // flowing again. - // So, if this is awaiting a drain, then we just call it now. - // If we don't know, then assume that we are waiting for one. - if (state.awaitDrain && (!dest._writableState || dest._writableState.needDrain)) ondrain(); - } - - // If the user pushes more data while we're writing to dest then we'll end up - // in ondata again. However, we only want to increase awaitDrain once because - // dest will only emit one 'drain' event for the multiple writes. - // => Introduce a guard on increasing awaitDrain. - var increasedAwaitDrain = false; - src.on('data', ondata); - function ondata(chunk) { - debug('ondata'); - increasedAwaitDrain = false; - var ret = dest.write(chunk); - if (false === ret && !increasedAwaitDrain) { - // If the user unpiped during `dest.write()`, it is possible - // to get stuck in a permanently paused state if that write - // also returned false. - // => Check whether `dest` is still a piping destination. - if ((state.pipesCount === 1 && state.pipes === dest || state.pipesCount > 1 && indexOf(state.pipes, dest) !== -1) && !cleanedUp) { - debug('false write response, pause', src._readableState.awaitDrain); - src._readableState.awaitDrain++; - increasedAwaitDrain = true; - } - src.pause(); - } - } - - // if the dest has an error, then stop piping into it. - // however, don't suppress the throwing behavior for this. - function onerror(er) { - debug('onerror', er); - unpipe(); - dest.removeListener('error', onerror); - if (listenerCount(dest, 'error') === 0) dest.emit('error', er); - } - - // Make sure our error handler is attached before userland ones. - prependListener(dest, 'error', onerror); - - // Both close and finish should trigger unpipe, but only once. - function onclose() { - dest.removeListener('finish', onfinish); - unpipe(); - } - dest.once('close', onclose); - function onfinish() { - debug('onfinish'); - dest.removeListener('close', onclose); - unpipe(); - } - dest.once('finish', onfinish); - - function unpipe() { - debug('unpipe'); - src.unpipe(dest); - } - - // tell the dest that it's being piped to - dest.emit('pipe', src); - - // start the flow if it hasn't been started already. - if (!state.flowing) { - debug('pipe resume'); - src.resume(); - } - - return dest; - }; - - function pipeOnDrain(src) { - return function () { - var state = src._readableState; - debug('pipeOnDrain', state.awaitDrain); - if (state.awaitDrain) state.awaitDrain--; - if (state.awaitDrain === 0 && src.listeners('data').length) { - state.flowing = true; - flow(src); - } - }; - } - - Readable.prototype.unpipe = function (dest) { - var state = this._readableState; - - // if we're not piping anywhere, then do nothing. - if (state.pipesCount === 0) return this; - - // just one destination. most common case. - if (state.pipesCount === 1) { - // passed in one, but it's not the right one. - if (dest && dest !== state.pipes) return this; - - if (!dest) dest = state.pipes; - - // got a match. - state.pipes = null; - state.pipesCount = 0; - state.flowing = false; - if (dest) dest.emit('unpipe', this); - return this; - } - - // slow case. multiple pipe destinations. - - if (!dest) { - // remove all. - var dests = state.pipes; - var len = state.pipesCount; - state.pipes = null; - state.pipesCount = 0; - state.flowing = false; - - for (var _i = 0; _i < len; _i++) { - dests[_i].emit('unpipe', this); - }return this; - } - - // try to find the right one. - var i = indexOf(state.pipes, dest); - if (i === -1) return this; - - state.pipes.splice(i, 1); - state.pipesCount -= 1; - if (state.pipesCount === 1) state.pipes = state.pipes[0]; - - dest.emit('unpipe', this); - - return this; - }; - - // set up data events if they are asked for - // Ensure readable listeners eventually get something - Readable.prototype.on = function (ev, fn) { - var res = EventEmitter.prototype.on.call(this, ev, fn); - - if (ev === 'data') { - // Start flowing on next tick if stream isn't explicitly paused - if (this._readableState.flowing !== false) this.resume(); - } else if (ev === 'readable') { - var state = this._readableState; - if (!state.endEmitted && !state.readableListening) { - state.readableListening = state.needReadable = true; - state.emittedReadable = false; - if (!state.reading) { - nextTick(nReadingNextTick, this); - } else if (state.length) { - emitReadable(this); - } - } - } - - return res; - }; - Readable.prototype.addListener = Readable.prototype.on; - - function nReadingNextTick(self) { - debug('readable nexttick read 0'); - self.read(0); - } - - // pause() and resume() are remnants of the legacy readable stream API - // If the user uses them, then switch into old mode. - Readable.prototype.resume = function () { - var state = this._readableState; - if (!state.flowing) { - debug('resume'); - state.flowing = true; - resume(this, state); - } - return this; - }; - - function resume(stream, state) { - if (!state.resumeScheduled) { - state.resumeScheduled = true; - nextTick(resume_, stream, state); - } - } - - function resume_(stream, state) { - if (!state.reading) { - debug('resume read 0'); - stream.read(0); - } - - state.resumeScheduled = false; - state.awaitDrain = 0; - stream.emit('resume'); - flow(stream); - if (state.flowing && !state.reading) stream.read(0); - } - - Readable.prototype.pause = function () { - debug('call pause flowing=%j', this._readableState.flowing); - if (false !== this._readableState.flowing) { - debug('pause'); - this._readableState.flowing = false; - this.emit('pause'); - } - return this; - }; - - function flow(stream) { - var state = stream._readableState; - debug('flow', state.flowing); - while (state.flowing && stream.read() !== null) {} - } - - // wrap an old-style stream as the async data source. - // This is *not* part of the readable stream interface. - // It is an ugly unfortunate mess of history. - Readable.prototype.wrap = function (stream) { - var state = this._readableState; - var paused = false; - - var self = this; - stream.on('end', function () { - debug('wrapped end'); - if (state.decoder && !state.ended) { - var chunk = state.decoder.end(); - if (chunk && chunk.length) self.push(chunk); - } - - self.push(null); - }); - - stream.on('data', function (chunk) { - debug('wrapped data'); - if (state.decoder) chunk = state.decoder.write(chunk); - - // don't skip over falsy values in objectMode - if (state.objectMode && (chunk === null || chunk === undefined)) return;else if (!state.objectMode && (!chunk || !chunk.length)) return; - - var ret = self.push(chunk); - if (!ret) { - paused = true; - stream.pause(); - } - }); - - // proxy all the other methods. - // important when wrapping filters and duplexes. - for (var i in stream) { - if (this[i] === undefined && typeof stream[i] === 'function') { - this[i] = function (method) { - return function () { - return stream[method].apply(stream, arguments); - }; - }(i); - } - } - - // proxy certain important events. - var events = ['error', 'close', 'destroy', 'pause', 'resume']; - forEach(events, function (ev) { - stream.on(ev, self.emit.bind(self, ev)); - }); - - // when we try to consume some more bytes, simply unpause the - // underlying stream. - self._read = function (n) { - debug('wrapped _read', n); - if (paused) { - paused = false; - stream.resume(); - } - }; - - return self; - }; - - // exposed for testing purposes only. - Readable._fromList = fromList; - - // Pluck off n bytes from an array of buffers. - // Length is the combined lengths of all the buffers in the list. - // This function is designed to be inlinable, so please take care when making - // changes to the function body. - function fromList(n, state) { - // nothing buffered - if (state.length === 0) return null; - - var ret; - if (state.objectMode) ret = state.buffer.shift();else if (!n || n >= state.length) { - // read it all, truncate the list - if (state.decoder) ret = state.buffer.join('');else if (state.buffer.length === 1) ret = state.buffer.head.data;else ret = state.buffer.concat(state.length); - state.buffer.clear(); - } else { - // read part of list - ret = fromListPartial(n, state.buffer, state.decoder); - } - - return ret; - } - - // Extracts only enough buffered data to satisfy the amount requested. - // This function is designed to be inlinable, so please take care when making - // changes to the function body. - function fromListPartial(n, list, hasStrings) { - var ret; - if (n < list.head.data.length) { - // slice is the same for buffers and strings - ret = list.head.data.slice(0, n); - list.head.data = list.head.data.slice(n); - } else if (n === list.head.data.length) { - // first chunk is a perfect match - ret = list.shift(); - } else { - // result spans more than one buffer - ret = hasStrings ? copyFromBufferString(n, list) : copyFromBuffer(n, list); - } - return ret; - } - - // Copies a specified amount of characters from the list of buffered data - // chunks. - // This function is designed to be inlinable, so please take care when making - // changes to the function body. - function copyFromBufferString(n, list) { - var p = list.head; - var c = 1; - var ret = p.data; - n -= ret.length; - while (p = p.next) { - var str = p.data; - var nb = n > str.length ? str.length : n; - if (nb === str.length) ret += str;else ret += str.slice(0, n); - n -= nb; - if (n === 0) { - if (nb === str.length) { - ++c; - if (p.next) list.head = p.next;else list.head = list.tail = null; - } else { - list.head = p; - p.data = str.slice(nb); - } - break; - } - ++c; - } - list.length -= c; - return ret; - } - - // Copies a specified amount of bytes from the list of buffered data chunks. - // This function is designed to be inlinable, so please take care when making - // changes to the function body. - function copyFromBuffer(n, list) { - var ret = Buffer.allocUnsafe(n); - var p = list.head; - var c = 1; - p.data.copy(ret); - n -= p.data.length; - while (p = p.next) { - var buf = p.data; - var nb = n > buf.length ? buf.length : n; - buf.copy(ret, ret.length - n, 0, nb); - n -= nb; - if (n === 0) { - if (nb === buf.length) { - ++c; - if (p.next) list.head = p.next;else list.head = list.tail = null; - } else { - list.head = p; - p.data = buf.slice(nb); - } - break; - } - ++c; - } - list.length -= c; - return ret; - } - - function endReadable(stream) { - var state = stream._readableState; - - // If we get here before consuming all the bytes, then that is a - // bug in node. Should never happen. - if (state.length > 0) throw new Error('"endReadable()" called on non-empty stream'); - - if (!state.endEmitted) { - state.ended = true; - nextTick(endReadableNT, state, stream); - } - } - - function endReadableNT(state, stream) { - // Check that we didn't get one last unshift. - if (!state.endEmitted && state.length === 0) { - state.endEmitted = true; - stream.readable = false; - stream.emit('end'); - } - } - - function forEach(xs, f) { - for (var i = 0, l = xs.length; i < l; i++) { - f(xs[i], i); - } - } - - function indexOf(xs, x) { - for (var i = 0, l = xs.length; i < l; i++) { - if (xs[i] === x) return i; - } - return -1; - } - - // A bit simpler than readable streams. - Writable.WritableState = WritableState; - inherits$1(Writable, EventEmitter); - - function nop() {} - - function WriteReq(chunk, encoding, cb) { - this.chunk = chunk; - this.encoding = encoding; - this.callback = cb; - this.next = null; - } - - function WritableState(options, stream) { - Object.defineProperty(this, 'buffer', { - get: deprecate(function () { - return this.getBuffer(); - }, '_writableState.buffer is deprecated. Use _writableState.getBuffer ' + 'instead.') - }); - options = options || {}; - - // object stream flag to indicate whether or not this stream - // contains buffers or objects. - this.objectMode = !!options.objectMode; - - if (stream instanceof Duplex) this.objectMode = this.objectMode || !!options.writableObjectMode; - - // the point at which write() starts returning false - // Note: 0 is a valid value, means that we always return false if - // the entire buffer is not flushed immediately on write() - var hwm = options.highWaterMark; - var defaultHwm = this.objectMode ? 16 : 16 * 1024; - this.highWaterMark = hwm || hwm === 0 ? hwm : defaultHwm; - - // cast to ints. - this.highWaterMark = ~ ~this.highWaterMark; - - this.needDrain = false; - // at the start of calling end() - this.ending = false; - // when end() has been called, and returned - this.ended = false; - // when 'finish' is emitted - this.finished = false; - - // should we decode strings into buffers before passing to _write? - // this is here so that some node-core streams can optimize string - // handling at a lower level. - var noDecode = options.decodeStrings === false; - this.decodeStrings = !noDecode; - - // Crypto is kind of old and crusty. Historically, its default string - // encoding is 'binary' so we have to make this configurable. - // Everything else in the universe uses 'utf8', though. - this.defaultEncoding = options.defaultEncoding || 'utf8'; - - // not an actual buffer we keep track of, but a measurement - // of how much we're waiting to get pushed to some underlying - // socket or file. - this.length = 0; - - // a flag to see when we're in the middle of a write. - this.writing = false; - - // when true all writes will be buffered until .uncork() call - this.corked = 0; - - // a flag to be able to tell if the onwrite cb is called immediately, - // or on a later tick. We set this to true at first, because any - // actions that shouldn't happen until "later" should generally also - // not happen before the first write call. - this.sync = true; - - // a flag to know if we're processing previously buffered items, which - // may call the _write() callback in the same tick, so that we don't - // end up in an overlapped onwrite situation. - this.bufferProcessing = false; - - // the callback that's passed to _write(chunk,cb) - this.onwrite = function (er) { - onwrite(stream, er); - }; - - // the callback that the user supplies to write(chunk,encoding,cb) - this.writecb = null; - - // the amount that is being written when _write is called. - this.writelen = 0; - - this.bufferedRequest = null; - this.lastBufferedRequest = null; - - // number of pending user-supplied write callbacks - // this must be 0 before 'finish' can be emitted - this.pendingcb = 0; - - // emit prefinish if the only thing we're waiting for is _write cbs - // This is relevant for synchronous Transform streams - this.prefinished = false; - - // True if the error was already emitted and should not be thrown again - this.errorEmitted = false; - - // count buffered requests - this.bufferedRequestCount = 0; - - // allocate the first CorkedRequest, there is always - // one allocated and free to use, and we maintain at most two - this.corkedRequestsFree = new CorkedRequest(this); - } - - WritableState.prototype.getBuffer = function writableStateGetBuffer() { - var current = this.bufferedRequest; - var out = []; - while (current) { - out.push(current); - current = current.next; - } - return out; - }; - function Writable(options) { - - // Writable ctor is applied to Duplexes, though they're not - // instanceof Writable, they're instanceof Readable. - if (!(this instanceof Writable) && !(this instanceof Duplex)) return new Writable(options); - - this._writableState = new WritableState(options, this); - - // legacy. - this.writable = true; - - if (options) { - if (typeof options.write === 'function') this._write = options.write; - - if (typeof options.writev === 'function') this._writev = options.writev; - } - - EventEmitter.call(this); - } - - // Otherwise people can pipe Writable streams, which is just wrong. - Writable.prototype.pipe = function () { - this.emit('error', new Error('Cannot pipe, not readable')); - }; - - function writeAfterEnd(stream, cb) { - var er = new Error('write after end'); - // TODO: defer error events consistently everywhere, not just the cb - stream.emit('error', er); - nextTick(cb, er); - } - - // If we get something that is not a buffer, string, null, or undefined, - // and we're not in objectMode, then that's an error. - // Otherwise stream chunks are all considered to be of length=1, and the - // watermarks determine how many objects to keep in the buffer, rather than - // how many bytes or characters. - function validChunk(stream, state, chunk, cb) { - var valid = true; - var er = false; - // Always throw error if a null is written - // if we are not in object mode then throw - // if it is not a buffer, string, or undefined. - if (chunk === null) { - er = new TypeError('May not write null values to stream'); - } else if (!Buffer.isBuffer(chunk) && typeof chunk !== 'string' && chunk !== undefined && !state.objectMode) { - er = new TypeError('Invalid non-string/buffer chunk'); - } - if (er) { - stream.emit('error', er); - nextTick(cb, er); - valid = false; - } - return valid; - } - - Writable.prototype.write = function (chunk, encoding, cb) { - var state = this._writableState; - var ret = false; - - if (typeof encoding === 'function') { - cb = encoding; - encoding = null; - } - - if (Buffer.isBuffer(chunk)) encoding = 'buffer';else if (!encoding) encoding = state.defaultEncoding; - - if (typeof cb !== 'function') cb = nop; - - if (state.ended) writeAfterEnd(this, cb);else if (validChunk(this, state, chunk, cb)) { - state.pendingcb++; - ret = writeOrBuffer(this, state, chunk, encoding, cb); - } - - return ret; - }; - - Writable.prototype.cork = function () { - var state = this._writableState; - - state.corked++; - }; - - Writable.prototype.uncork = function () { - var state = this._writableState; - - if (state.corked) { - state.corked--; - - if (!state.writing && !state.corked && !state.finished && !state.bufferProcessing && state.bufferedRequest) clearBuffer(this, state); - } - }; - - Writable.prototype.setDefaultEncoding = function setDefaultEncoding(encoding) { - // node::ParseEncoding() requires lower case. - if (typeof encoding === 'string') encoding = encoding.toLowerCase(); - if (!(['hex', 'utf8', 'utf-8', 'ascii', 'binary', 'base64', 'ucs2', 'ucs-2', 'utf16le', 'utf-16le', 'raw'].indexOf((encoding + '').toLowerCase()) > -1)) throw new TypeError('Unknown encoding: ' + encoding); - this._writableState.defaultEncoding = encoding; - return this; - }; - - function decodeChunk(state, chunk, encoding) { - if (!state.objectMode && state.decodeStrings !== false && typeof chunk === 'string') { - chunk = Buffer.from(chunk, encoding); - } - return chunk; - } - - // if we're already writing something, then just put this - // in the queue, and wait our turn. Otherwise, call _write - // If we return false, then we need a drain event, so set that flag. - function writeOrBuffer(stream, state, chunk, encoding, cb) { - chunk = decodeChunk(state, chunk, encoding); - - if (Buffer.isBuffer(chunk)) encoding = 'buffer'; - var len = state.objectMode ? 1 : chunk.length; - - state.length += len; - - var ret = state.length < state.highWaterMark; - // we must ensure that previous needDrain will not be reset to false. - if (!ret) state.needDrain = true; - - if (state.writing || state.corked) { - var last = state.lastBufferedRequest; - state.lastBufferedRequest = new WriteReq(chunk, encoding, cb); - if (last) { - last.next = state.lastBufferedRequest; - } else { - state.bufferedRequest = state.lastBufferedRequest; - } - state.bufferedRequestCount += 1; - } else { - doWrite(stream, state, false, len, chunk, encoding, cb); - } - - return ret; - } - - function doWrite(stream, state, writev, len, chunk, encoding, cb) { - state.writelen = len; - state.writecb = cb; - state.writing = true; - state.sync = true; - if (writev) stream._writev(chunk, state.onwrite);else stream._write(chunk, encoding, state.onwrite); - state.sync = false; - } - - function onwriteError(stream, state, sync, er, cb) { - --state.pendingcb; - if (sync) nextTick(cb, er);else cb(er); - - stream._writableState.errorEmitted = true; - stream.emit('error', er); - } - - function onwriteStateUpdate(state) { - state.writing = false; - state.writecb = null; - state.length -= state.writelen; - state.writelen = 0; - } - - function onwrite(stream, er) { - var state = stream._writableState; - var sync = state.sync; - var cb = state.writecb; - - onwriteStateUpdate(state); - - if (er) onwriteError(stream, state, sync, er, cb);else { - // Check if we're actually ready to finish, but don't emit yet - var finished = needFinish(state); - - if (!finished && !state.corked && !state.bufferProcessing && state.bufferedRequest) { - clearBuffer(stream, state); - } - - if (sync) { - /**/ - nextTick(afterWrite, stream, state, finished, cb); - /**/ - } else { - afterWrite(stream, state, finished, cb); - } - } - } - - function afterWrite(stream, state, finished, cb) { - if (!finished) onwriteDrain(stream, state); - state.pendingcb--; - cb(); - finishMaybe(stream, state); - } - - // Must force callback to be called on nextTick, so that we don't - // emit 'drain' before the write() consumer gets the 'false' return - // value, and has a chance to attach a 'drain' listener. - function onwriteDrain(stream, state) { - if (state.length === 0 && state.needDrain) { - state.needDrain = false; - stream.emit('drain'); - } - } - - // if there's something in the buffer waiting, then process it - function clearBuffer(stream, state) { - state.bufferProcessing = true; - var entry = state.bufferedRequest; - - if (stream._writev && entry && entry.next) { - // Fast case, write everything using _writev() - var l = state.bufferedRequestCount; - var buffer = new Array(l); - var holder = state.corkedRequestsFree; - holder.entry = entry; - - var count = 0; - while (entry) { - buffer[count] = entry; - entry = entry.next; - count += 1; - } - - doWrite(stream, state, true, state.length, buffer, '', holder.finish); - - // doWrite is almost always async, defer these to save a bit of time - // as the hot path ends with doWrite - state.pendingcb++; - state.lastBufferedRequest = null; - if (holder.next) { - state.corkedRequestsFree = holder.next; - holder.next = null; - } else { - state.corkedRequestsFree = new CorkedRequest(state); - } - } else { - // Slow case, write chunks one-by-one - while (entry) { - var chunk = entry.chunk; - var encoding = entry.encoding; - var cb = entry.callback; - var len = state.objectMode ? 1 : chunk.length; - - doWrite(stream, state, false, len, chunk, encoding, cb); - entry = entry.next; - // if we didn't call the onwrite immediately, then - // it means that we need to wait until it does. - // also, that means that the chunk and cb are currently - // being processed, so move the buffer counter past them. - if (state.writing) { - break; - } - } - - if (entry === null) state.lastBufferedRequest = null; - } - - state.bufferedRequestCount = 0; - state.bufferedRequest = entry; - state.bufferProcessing = false; - } - - Writable.prototype._write = function (chunk, encoding, cb) { - cb(new Error('not implemented')); - }; - - Writable.prototype._writev = null; - - Writable.prototype.end = function (chunk, encoding, cb) { - var state = this._writableState; - - if (typeof chunk === 'function') { - cb = chunk; - chunk = null; - encoding = null; - } else if (typeof encoding === 'function') { - cb = encoding; - encoding = null; - } - - if (chunk !== null && chunk !== undefined) this.write(chunk, encoding); - - // .end() fully uncorks - if (state.corked) { - state.corked = 1; - this.uncork(); - } - - // ignore unnecessary end() calls. - if (!state.ending && !state.finished) endWritable(this, state, cb); - }; - - function needFinish(state) { - return state.ending && state.length === 0 && state.bufferedRequest === null && !state.finished && !state.writing; - } - - function prefinish(stream, state) { - if (!state.prefinished) { - state.prefinished = true; - stream.emit('prefinish'); - } - } - - function finishMaybe(stream, state) { - var need = needFinish(state); - if (need) { - if (state.pendingcb === 0) { - prefinish(stream, state); - state.finished = true; - stream.emit('finish'); - } else { - prefinish(stream, state); - } - } - return need; - } - - function endWritable(stream, state, cb) { - state.ending = true; - finishMaybe(stream, state); - if (cb) { - if (state.finished) nextTick(cb);else stream.once('finish', cb); - } - state.ended = true; - stream.writable = false; - } - - // It seems a linked list but it is not - // there will be only 2 of these for each stream - function CorkedRequest(state) { - var _this = this; - - this.next = null; - this.entry = null; - - this.finish = function (err) { - var entry = _this.entry; - _this.entry = null; - while (entry) { - var cb = entry.callback; - state.pendingcb--; - cb(err); - entry = entry.next; - } - if (state.corkedRequestsFree) { - state.corkedRequestsFree.next = _this; - } else { - state.corkedRequestsFree = _this; - } - }; - } - - inherits$1(Duplex, Readable); - - var keys = Object.keys(Writable.prototype); - for (var v = 0; v < keys.length; v++) { - var method = keys[v]; - if (!Duplex.prototype[method]) Duplex.prototype[method] = Writable.prototype[method]; - } - function Duplex(options) { - if (!(this instanceof Duplex)) return new Duplex(options); - - Readable.call(this, options); - Writable.call(this, options); - - if (options && options.readable === false) this.readable = false; - - if (options && options.writable === false) this.writable = false; - - this.allowHalfOpen = true; - if (options && options.allowHalfOpen === false) this.allowHalfOpen = false; - - this.once('end', onend); - } - - // the no-half-open enforcer - function onend() { - // if we allow half-open state, or if the writable side ended, - // then we're ok. - if (this.allowHalfOpen || this._writableState.ended) return; - - // no more data can be written. - // But allow more writes to happen in this tick. - nextTick(onEndNT, this); - } - - function onEndNT(self) { - self.end(); - } - - // a transform stream is a readable/writable stream where you do - inherits$1(Transform, Duplex); - - function TransformState(stream) { - this.afterTransform = function (er, data) { - return afterTransform(stream, er, data); - }; - - this.needTransform = false; - this.transforming = false; - this.writecb = null; - this.writechunk = null; - this.writeencoding = null; - } - - function afterTransform(stream, er, data) { - var ts = stream._transformState; - ts.transforming = false; - - var cb = ts.writecb; - - if (!cb) return stream.emit('error', new Error('no writecb in Transform class')); - - ts.writechunk = null; - ts.writecb = null; - - if (data !== null && data !== undefined) stream.push(data); - - cb(er); - - var rs = stream._readableState; - rs.reading = false; - if (rs.needReadable || rs.length < rs.highWaterMark) { - stream._read(rs.highWaterMark); - } - } - function Transform(options) { - if (!(this instanceof Transform)) return new Transform(options); - - Duplex.call(this, options); - - this._transformState = new TransformState(this); - - // when the writable side finishes, then flush out anything remaining. - var stream = this; - - // start out asking for a readable event once data is transformed. - this._readableState.needReadable = true; - - // we have implemented the _read method, and done the other things - // that Readable wants before the first _read call, so unset the - // sync guard flag. - this._readableState.sync = false; - - if (options) { - if (typeof options.transform === 'function') this._transform = options.transform; - - if (typeof options.flush === 'function') this._flush = options.flush; - } - - this.once('prefinish', function () { - if (typeof this._flush === 'function') this._flush(function (er) { - done(stream, er); - });else done(stream); - }); - } - - Transform.prototype.push = function (chunk, encoding) { - this._transformState.needTransform = false; - return Duplex.prototype.push.call(this, chunk, encoding); - }; - - // This is the part where you do stuff! - // override this function in implementation classes. - // 'chunk' is an input chunk. - // - // Call `push(newChunk)` to pass along transformed output - // to the readable side. You may call 'push' zero or more times. - // - // Call `cb(err)` when you are done with this chunk. If you pass - // an error, then that'll put the hurt on the whole operation. If you - // never call cb(), then you'll never get another chunk. - Transform.prototype._transform = function (chunk, encoding, cb) { - throw new Error('Not implemented'); - }; - - Transform.prototype._write = function (chunk, encoding, cb) { - var ts = this._transformState; - ts.writecb = cb; - ts.writechunk = chunk; - ts.writeencoding = encoding; - if (!ts.transforming) { - var rs = this._readableState; - if (ts.needTransform || rs.needReadable || rs.length < rs.highWaterMark) this._read(rs.highWaterMark); - } - }; - - // Doesn't matter what the args are here. - // _transform does all the work. - // That we got here means that the readable side wants more data. - Transform.prototype._read = function (n) { - var ts = this._transformState; - - if (ts.writechunk !== null && ts.writecb && !ts.transforming) { - ts.transforming = true; - this._transform(ts.writechunk, ts.writeencoding, ts.afterTransform); - } else { - // mark that we need a transform, so that any data that comes in - // will get processed, now that we've asked for it. - ts.needTransform = true; - } - }; - - function done(stream, er) { - if (er) return stream.emit('error', er); - - // if there's nothing in the write buffer, then that means - // that nothing more will ever be provided - var ws = stream._writableState; - var ts = stream._transformState; - - if (ws.length) throw new Error('Calling transform done when ws.length != 0'); - - if (ts.transforming) throw new Error('Calling transform done when still transforming'); - - return stream.push(null); - } - - inherits$1(PassThrough, Transform); - function PassThrough(options) { - if (!(this instanceof PassThrough)) return new PassThrough(options); - - Transform.call(this, options); - } - - PassThrough.prototype._transform = function (chunk, encoding, cb) { - cb(null, chunk); - }; - - inherits$1(Stream, EventEmitter); - Stream.Readable = Readable; - Stream.Writable = Writable; - Stream.Duplex = Duplex; - Stream.Transform = Transform; - Stream.PassThrough = PassThrough; - - // Backwards-compat with node 0.4.x - Stream.Stream = Stream; - - // old-style streams. Note that the pipe method (the only relevant - // part of this class) is overridden in the Readable class. - - function Stream() { - EventEmitter.call(this); - } - - Stream.prototype.pipe = function(dest, options) { - var source = this; - - function ondata(chunk) { - if (dest.writable) { - if (false === dest.write(chunk) && source.pause) { - source.pause(); - } - } - } - - source.on('data', ondata); - - function ondrain() { - if (source.readable && source.resume) { - source.resume(); - } - } - - dest.on('drain', ondrain); - - // If the 'end' option is not supplied, dest.end() will be called when - // source gets the 'end' or 'close' events. Only dest.end() once. - if (!dest._isStdio && (!options || options.end !== false)) { - source.on('end', onend); - source.on('close', onclose); - } - - var didOnEnd = false; - function onend() { - if (didOnEnd) return; - didOnEnd = true; - - dest.end(); - } - - - function onclose() { - if (didOnEnd) return; - didOnEnd = true; - - if (typeof dest.destroy === 'function') dest.destroy(); - } - - // don't leave dangling pipes when there are errors. - function onerror(er) { - cleanup(); - if (EventEmitter.listenerCount(this, 'error') === 0) { - throw er; // Unhandled stream error in pipe. - } - } - - source.on('error', onerror); - dest.on('error', onerror); - - // remove all the event listeners that were added. - function cleanup() { - source.removeListener('data', ondata); - dest.removeListener('drain', ondrain); - - source.removeListener('end', onend); - source.removeListener('close', onclose); - - source.removeListener('error', onerror); - dest.removeListener('error', onerror); - - source.removeListener('end', cleanup); - source.removeListener('close', cleanup); - - dest.removeListener('close', cleanup); - } - - source.on('end', cleanup); - source.on('close', cleanup); - - dest.on('close', cleanup); - - dest.emit('pipe', source); - - // Allow for unix-like usage: A.pipe(B).pipe(C) - return dest; - }; - - const bom_utf8 = Buffer.from([239, 187, 191]); - - class CsvError extends Error { - constructor(code, message, ...contexts) { - if(Array.isArray(message)) message = message.join(' '); - super(message); - if(Error.captureStackTrace !== undefined){ - Error.captureStackTrace(this, CsvError); - } - this.code = code; - for(const context of contexts){ - for(const key in context){ - const value = context[key]; - this[key] = isBuffer(value) ? value.toString() : value == null ? value : JSON.parse(JSON.stringify(value)); - } - } - } - } - - const isObject = function(obj){ - return typeof obj === 'object' && obj !== null && ! Array.isArray(obj); - }; - - const underscore = function(str){ - return str.replace(/([A-Z])/g, function(_, match){ - return '_' + match.toLowerCase(); - }); - }; - // Lodash implementation of `get` const charCodeOfDot = '.'.charCodeAt(0); @@ -5075,455 +2043,473 @@ var csv_stringify_sync = (function (exports) { return (index && index === length) ? object : undefined; }; - class Stringifier extends Transform { - constructor(opts = {}){ - super({...{writableObjectMode: true}, ...opts}); - const options = {}; - let err; - // Merge with user options - for(const opt in opts){ - options[underscore(opt)] = opts[opt]; - } - if((err = this.normalize(options)) !== undefined) throw err; - switch(options.record_delimiter){ - case 'auto': - options.record_delimiter = null; - break; - case 'unix': - options.record_delimiter = "\n"; - break; - case 'mac': - options.record_delimiter = "\r"; - break; - case 'windows': - options.record_delimiter = "\r\n"; - break; - case 'ascii': - options.record_delimiter = "\u001e"; - break; - case 'unicode': - options.record_delimiter = "\u2028"; - break; - } - // Expose options - this.options = options; - // Internal state - this.state = { - stop: false - }; - // Information - this.info = { - records: 0 - }; - } - normalize(options){ - // Normalize option `bom` - if(options.bom === undefined || options.bom === null || options.bom === false){ - options.bom = false; - }else if(options.bom !== true){ - return new CsvError('CSV_OPTION_BOOLEAN_INVALID_TYPE', [ - 'option `bom` is optional and must be a boolean value,', - `got ${JSON.stringify(options.bom)}` - ]); - } - // Normalize option `delimiter` - if(options.delimiter === undefined || options.delimiter === null){ - options.delimiter = ','; - }else if(isBuffer(options.delimiter)){ - options.delimiter = options.delimiter.toString(); - }else if(typeof options.delimiter !== 'string'){ - return new CsvError('CSV_OPTION_DELIMITER_INVALID_TYPE', [ - 'option `delimiter` must be a buffer or a string,', - `got ${JSON.stringify(options.delimiter)}` - ]); - } - // Normalize option `quote` - if(options.quote === undefined || options.quote === null){ - options.quote = '"'; - }else if(options.quote === true){ - options.quote = '"'; - }else if(options.quote === false){ - options.quote = ''; - }else if (isBuffer(options.quote)){ - options.quote = options.quote.toString(); - }else if(typeof options.quote !== 'string'){ - return new CsvError('CSV_OPTION_QUOTE_INVALID_TYPE', [ - 'option `quote` must be a boolean, a buffer or a string,', - `got ${JSON.stringify(options.quote)}` - ]); - } - // Normalize option `quoted` - if(options.quoted === undefined || options.quoted === null){ - options.quoted = false; - } - // Normalize option `quoted_empty` - if(options.quoted_empty === undefined || options.quoted_empty === null){ - options.quoted_empty = undefined; - } - // Normalize option `quoted_match` - if(options.quoted_match === undefined || options.quoted_match === null || options.quoted_match === false){ - options.quoted_match = null; - }else if(!Array.isArray(options.quoted_match)){ - options.quoted_match = [options.quoted_match]; + const is_object = function(obj){ + return typeof obj === 'object' && obj !== null && ! Array.isArray(obj); + }; + + const normalize_columns = function(columns){ + if(columns === undefined || columns === null){ + return [undefined, undefined]; + } + if(typeof columns !== 'object'){ + return [Error('Invalid option "columns": expect an array or an object')]; + } + if(!Array.isArray(columns)){ + const newcolumns = []; + for(const k in columns){ + newcolumns.push({ + key: k, + header: columns[k] + }); } - if(options.quoted_match){ - for(const quoted_match of options.quoted_match){ - const isString = typeof quoted_match === 'string'; - const isRegExp = quoted_match instanceof RegExp; - if(!isString && !isRegExp){ - return Error(`Invalid Option: quoted_match must be a string or a regex, got ${JSON.stringify(quoted_match)}`); + columns = newcolumns; + }else { + const newcolumns = []; + for(const column of columns){ + if(typeof column === 'string'){ + newcolumns.push({ + key: column, + header: column + }); + }else if(typeof column === 'object' && column !== null && !Array.isArray(column)){ + if(!column.key){ + return [Error('Invalid column definition: property "key" is required')]; + } + if(column.header === undefined){ + column.header = column.key; } + newcolumns.push(column); + }else { + return [Error('Invalid column definition: expect a string or an object')]; } } - // Normalize option `quoted_string` - if(options.quoted_string === undefined || options.quoted_string === null){ - options.quoted_string = false; - } - // Normalize option `eof` - if(options.eof === undefined || options.eof === null){ - options.eof = true; - } - // Normalize option `escape` - if(options.escape === undefined || options.escape === null){ - options.escape = '"'; - }else if(isBuffer(options.escape)){ - options.escape = options.escape.toString(); - }else if(typeof options.escape !== 'string'){ - return Error(`Invalid Option: escape must be a buffer or a string, got ${JSON.stringify(options.escape)}`); - } - if (options.escape.length > 1){ - return Error(`Invalid Option: escape must be one character, got ${options.escape.length} characters`); - } - // Normalize option `header` - if(options.header === undefined || options.header === null){ - options.header = false; - } - // Normalize option `columns` - const [err, columns] = this.normalize_columns(options.columns); - if(err) return err; - options.columns = columns; - // Normalize option `quoted` - if(options.quoted === undefined || options.quoted === null){ - options.quoted = false; - } - // Normalize option `cast` - if(options.cast === undefined || options.cast === null){ - options.cast = {}; - } - // Normalize option cast.bigint - if(options.cast.bigint === undefined || options.cast.bigint === null){ - // Cast boolean to string by default - options.cast.bigint = value => '' + value; - } - // Normalize option cast.boolean - if(options.cast.boolean === undefined || options.cast.boolean === null){ - // Cast boolean to string by default - options.cast.boolean = value => value ? '1' : ''; - } - // Normalize option cast.date - if(options.cast.date === undefined || options.cast.date === null){ - // Cast date to timestamp string by default - options.cast.date = value => '' + value.getTime(); - } - // Normalize option cast.number - if(options.cast.number === undefined || options.cast.number === null){ - // Cast number to string using native casting by default - options.cast.number = value => '' + value; - } - // Normalize option cast.object - if(options.cast.object === undefined || options.cast.object === null){ - // Stringify object as JSON by default - options.cast.object = value => JSON.stringify(value); - } - // Normalize option cast.string - if(options.cast.string === undefined || options.cast.string === null){ - // Leave string untouched - options.cast.string = function(value){return value;}; - } - // Normalize option `record_delimiter` - if(options.record_delimiter === undefined || options.record_delimiter === null){ - options.record_delimiter = '\n'; - }else if(isBuffer(options.record_delimiter)){ - options.record_delimiter = options.record_delimiter.toString(); - }else if(typeof options.record_delimiter !== 'string'){ - return Error(`Invalid Option: record_delimiter must be a buffer or a string, got ${JSON.stringify(options.record_delimiter)}`); - } + columns = newcolumns; } - _transform(chunk, encoding, callback){ - if(this.state.stop === true){ - return; + return [undefined, columns]; + }; + + class CsvError extends Error { + constructor(code, message, ...contexts) { + if(Array.isArray(message)) message = message.join(' '); + super(message); + if(Error.captureStackTrace !== undefined){ + Error.captureStackTrace(this, CsvError); } - const err = this.__transform(chunk); - if(err !== undefined){ - this.state.stop = true; + this.code = code; + for(const context of contexts){ + for(const key in context){ + const value = context[key]; + this[key] = isBuffer(value) ? value.toString() : value == null ? value : JSON.parse(JSON.stringify(value)); + } } - callback(err); } - _flush(callback){ - if(this.state.stop === true){ - // Note, Node.js 12 call flush even after an error, we must prevent - // `callback` from being called in flush without any error. - return; - } - if(this.info.records === 0){ - this.bom(); - const err = this.headers(); - if(err) callback(err); + } + + const underscore = function(str){ + return str.replace(/([A-Z])/g, function(_, match){ + return '_' + match.toLowerCase(); + }); + }; + + const normalize_options = function(opts) { + const options = {}; + // Merge with user options + for(const opt in opts){ + options[underscore(opt)] = opts[opt]; + } + // Normalize option `bom` + if(options.bom === undefined || options.bom === null || options.bom === false){ + options.bom = false; + }else if(options.bom !== true){ + return [new CsvError('CSV_OPTION_BOOLEAN_INVALID_TYPE', [ + 'option `bom` is optional and must be a boolean value,', + `got ${JSON.stringify(options.bom)}` + ])]; + } + // Normalize option `delimiter` + if(options.delimiter === undefined || options.delimiter === null){ + options.delimiter = ','; + }else if(isBuffer(options.delimiter)){ + options.delimiter = options.delimiter.toString(); + }else if(typeof options.delimiter !== 'string'){ + return [new CsvError('CSV_OPTION_DELIMITER_INVALID_TYPE', [ + 'option `delimiter` must be a buffer or a string,', + `got ${JSON.stringify(options.delimiter)}` + ])]; + } + // Normalize option `quote` + if(options.quote === undefined || options.quote === null){ + options.quote = '"'; + }else if(options.quote === true){ + options.quote = '"'; + }else if(options.quote === false){ + options.quote = ''; + }else if (isBuffer(options.quote)){ + options.quote = options.quote.toString(); + }else if(typeof options.quote !== 'string'){ + return [new CsvError('CSV_OPTION_QUOTE_INVALID_TYPE', [ + 'option `quote` must be a boolean, a buffer or a string,', + `got ${JSON.stringify(options.quote)}` + ])]; + } + // Normalize option `quoted` + if(options.quoted === undefined || options.quoted === null){ + options.quoted = false; + } + // Normalize option `quoted_empty` + if(options.quoted_empty === undefined || options.quoted_empty === null){ + options.quoted_empty = undefined; + } + // Normalize option `quoted_match` + if(options.quoted_match === undefined || options.quoted_match === null || options.quoted_match === false){ + options.quoted_match = null; + }else if(!Array.isArray(options.quoted_match)){ + options.quoted_match = [options.quoted_match]; + } + if(options.quoted_match){ + for(const quoted_match of options.quoted_match){ + const isString = typeof quoted_match === 'string'; + const isRegExp = quoted_match instanceof RegExp; + if(!isString && !isRegExp){ + return [Error(`Invalid Option: quoted_match must be a string or a regex, got ${JSON.stringify(quoted_match)}`)]; + } } - callback(); } - __transform(chunk){ - // Chunk validation - if(!Array.isArray(chunk) && typeof chunk !== 'object'){ - return Error(`Invalid Record: expect an array or an object, got ${JSON.stringify(chunk)}`); - } - // Detect columns from the first record - if(this.info.records === 0){ - if(Array.isArray(chunk)){ - if(this.options.header === true && this.options.columns === undefined){ - return Error('Undiscoverable Columns: header option requires column option or object records'); + // Normalize option `quoted_string` + if(options.quoted_string === undefined || options.quoted_string === null){ + options.quoted_string = false; + } + // Normalize option `eof` + if(options.eof === undefined || options.eof === null){ + options.eof = true; + } + // Normalize option `escape` + if(options.escape === undefined || options.escape === null){ + options.escape = '"'; + }else if(isBuffer(options.escape)){ + options.escape = options.escape.toString(); + }else if(typeof options.escape !== 'string'){ + return [Error(`Invalid Option: escape must be a buffer or a string, got ${JSON.stringify(options.escape)}`)]; + } + if (options.escape.length > 1){ + return [Error(`Invalid Option: escape must be one character, got ${options.escape.length} characters`)]; + } + // Normalize option `header` + if(options.header === undefined || options.header === null){ + options.header = false; + } + // Normalize option `columns` + const [errColumns, columns] = normalize_columns(options.columns); + if(errColumns !== undefined) return [errColumns]; + options.columns = columns; + // Normalize option `quoted` + if(options.quoted === undefined || options.quoted === null){ + options.quoted = false; + } + // Normalize option `cast` + if(options.cast === undefined || options.cast === null){ + options.cast = {}; + } + // Normalize option cast.bigint + if(options.cast.bigint === undefined || options.cast.bigint === null){ + // Cast boolean to string by default + options.cast.bigint = value => '' + value; + } + // Normalize option cast.boolean + if(options.cast.boolean === undefined || options.cast.boolean === null){ + // Cast boolean to string by default + options.cast.boolean = value => value ? '1' : ''; + } + // Normalize option cast.date + if(options.cast.date === undefined || options.cast.date === null){ + // Cast date to timestamp string by default + options.cast.date = value => '' + value.getTime(); + } + // Normalize option cast.number + if(options.cast.number === undefined || options.cast.number === null){ + // Cast number to string using native casting by default + options.cast.number = value => '' + value; + } + // Normalize option cast.object + if(options.cast.object === undefined || options.cast.object === null){ + // Stringify object as JSON by default + options.cast.object = value => JSON.stringify(value); + } + // Normalize option cast.string + if(options.cast.string === undefined || options.cast.string === null){ + // Leave string untouched + options.cast.string = function(value){return value;}; + } + // Normalize option `on_record` + if(options.on_record !== undefined && typeof options.on_record !== 'function'){ + return [Error(`Invalid Option: "on_record" must be a function.`)]; + } + // Normalize option `record_delimiter` + if(options.record_delimiter === undefined || options.record_delimiter === null){ + options.record_delimiter = '\n'; + }else if(isBuffer(options.record_delimiter)){ + options.record_delimiter = options.record_delimiter.toString(); + }else if(typeof options.record_delimiter !== 'string'){ + return [Error(`Invalid Option: record_delimiter must be a buffer or a string, got ${JSON.stringify(options.record_delimiter)}`)]; + } + switch(options.record_delimiter){ + case 'auto': + options.record_delimiter = null; + break; + case 'unix': + options.record_delimiter = "\n"; + break; + case 'mac': + options.record_delimiter = "\r"; + break; + case 'windows': + options.record_delimiter = "\r\n"; + break; + case 'ascii': + options.record_delimiter = "\u001e"; + break; + case 'unicode': + options.record_delimiter = "\u2028"; + break; + } + return [undefined, options]; + }; + + const bom_utf8 = Buffer.from([239, 187, 191]); + + const stringifier = function(options, state, info){ + return { + options: options, + state: state, + info: info, + __transform: function(chunk, push){ + // Chunk validation + if(!Array.isArray(chunk) && typeof chunk !== 'object'){ + return Error(`Invalid Record: expect an array or an object, got ${JSON.stringify(chunk)}`); + } + // Detect columns from the first record + if(this.info.records === 0){ + if(Array.isArray(chunk)){ + if(this.options.header === true && this.options.columns === undefined){ + return Error('Undiscoverable Columns: header option requires column option or object records'); + } + }else if(this.options.columns === undefined){ + const [err, columns] = normalize_columns(Object.keys(chunk)); + if(err) return; + this.options.columns = columns; } - }else if(this.options.columns === undefined){ - const [err, columns] = this.normalize_columns(Object.keys(chunk)); - if(err) return; - this.options.columns = columns; } - } - // Emit the header - if(this.info.records === 0){ - this.bom(); - const err = this.headers(); - if(err) return err; - } - // Emit and stringify the record if an object or an array - try{ - this.emit('record', chunk, this.info.records); - }catch(err){ - return err; - } - // Convert the record into a string - let err, chunk_string; - if(this.options.eof){ - [err, chunk_string] = this.stringify(chunk); - if(err) return err; - if(chunk_string === undefined){ - return; - }else { - chunk_string = chunk_string + this.options.record_delimiter; + // Emit the header + if(this.info.records === 0){ + this.bom(push); + const err = this.headers(push); + if(err) return err; } - }else { - [err, chunk_string] = this.stringify(chunk); - if(err) return err; - if(chunk_string === undefined){ - return; - }else { - if(this.options.header || this.info.records){ - chunk_string = this.options.record_delimiter + chunk_string; + // Emit and stringify the record if an object or an array + try{ + // this.emit('record', chunk, this.info.records); + if(this.options.on_record){ + this.options.on_record(chunk, this.info.records); } + }catch(err){ + return err; } - } - // Emit the csv - this.info.records++; - this.push(chunk_string); - } - stringify(chunk, chunkIsHeader=false){ - if(typeof chunk !== 'object'){ - return [undefined, chunk]; - } - const {columns} = this.options; - const record = []; - // Record is an array - if(Array.isArray(chunk)){ - // We are getting an array but the user has specified output columns. In - // this case, we respect the columns indexes - if(columns){ - chunk.splice(columns.length); - } - // Cast record elements - for(let i=0; i= 0; - const containsQuote = (quote !== '') && value.indexOf(quote) >= 0; - const containsEscape = value.indexOf(escape) >= 0 && (escape !== quote); - const containsRecordDelimiter = value.indexOf(record_delimiter) >= 0; - const quotedString = quoted_string && typeof field === 'string'; - let quotedMatch = quoted_match && quoted_match.filter(quoted_match => { - if(typeof quoted_match === 'string'){ - return value.indexOf(quoted_match) !== -1; - }else { - return quoted_match.test(value); + } + let csvrecord = ''; + for(let i=0; i 0; - const shouldQuote = containsQuote === true || containsdelimiter || containsRecordDelimiter || quoted || quotedString || quotedMatch; - if(shouldQuote === true && containsEscape === true){ - const regexp = escape === '\\' - ? new RegExp(escape + escape, 'g') - : new RegExp(escape, 'g'); - value = value.replace(regexp, escape + escape); + options = {...this.options, ...options}; + [err, options] = normalize_options(options); + if(err !== undefined){ + return [err]; + } + }else if(value === undefined || value === null){ + options = this.options; + }else { + return [Error(`Invalid Casting Value: returned value must return a string, an object, null or undefined, got ${JSON.stringify(value)}`)]; } - if(containsQuote === true){ - const regexp = new RegExp(quote,'g'); - value = value.replace(regexp, escape + quote); + const {delimiter, escape, quote, quoted, quoted_empty, quoted_string, quoted_match, record_delimiter} = options; + if(value){ + if(typeof value !== 'string'){ + return [Error(`Formatter must return a string, null or undefined, got ${JSON.stringify(value)}`)]; + } + const containsdelimiter = delimiter.length && value.indexOf(delimiter) >= 0; + const containsQuote = (quote !== '') && value.indexOf(quote) >= 0; + const containsEscape = value.indexOf(escape) >= 0 && (escape !== quote); + const containsRecordDelimiter = value.indexOf(record_delimiter) >= 0; + const quotedString = quoted_string && typeof field === 'string'; + let quotedMatch = quoted_match && quoted_match.filter(quoted_match => { + if(typeof quoted_match === 'string'){ + return value.indexOf(quoted_match) !== -1; + }else { + return quoted_match.test(value); + } + }); + quotedMatch = quotedMatch && quotedMatch.length > 0; + const shouldQuote = containsQuote === true || containsdelimiter || containsRecordDelimiter || quoted || quotedString || quotedMatch; + if(shouldQuote === true && containsEscape === true){ + const regexp = escape === '\\' + ? new RegExp(escape + escape, 'g') + : new RegExp(escape, 'g'); + value = value.replace(regexp, escape + escape); + } + if(containsQuote === true){ + const regexp = new RegExp(quote,'g'); + value = value.replace(regexp, escape + quote); + } + if(shouldQuote === true){ + value = quote + value + quote; + } + csvrecord += value; + }else if(quoted_empty === true || (field === '' && quoted_string === true && quoted_empty !== false)){ + csvrecord += quote + quote; } - if(shouldQuote === true){ - value = quote + value + quote; + if(i !== record.length - 1){ + csvrecord += delimiter; } - csvrecord += value; - }else if(quoted_empty === true || (field === '' && quoted_string === true && quoted_empty !== false)){ - csvrecord += quote + quote; } - if(i !== record.length - 1){ - csvrecord += delimiter; + return [undefined, csvrecord]; + }, + bom: function(push){ + if(this.options.bom !== true){ + return; } - } - return [undefined, csvrecord]; - } - bom(){ - if(this.options.bom !== true){ - return; - } - this.push(bom_utf8); - } - headers(){ - if(this.options.header === false){ - return; - } - if(this.options.columns === undefined){ - return; - } - let err; - let headers = this.options.columns.map(column => column.header); - if(this.options.eof){ - [err, headers] = this.stringify(headers, true); - headers += this.options.record_delimiter; - }else { - [err, headers] = this.stringify(headers); - } - if(err) return err; - this.push(headers); - } - __cast(value, context){ - const type = typeof value; - try{ - if(type === 'string'){ // Fine for 99% of the cases - return [undefined, this.options.cast.string(value, context)]; - }else if(type === 'bigint'){ - return [undefined, this.options.cast.bigint(value, context)]; - }else if(type === 'number'){ - return [undefined, this.options.cast.number(value, context)]; - }else if(type === 'boolean'){ - return [undefined, this.options.cast.boolean(value, context)]; - }else if(value instanceof Date){ - return [undefined, this.options.cast.date(value, context)]; - }else if(type === 'object' && value !== null){ - return [undefined, this.options.cast.object(value, context)]; - }else { - return [undefined, value, value]; + push(bom_utf8); + }, + headers: function(push){ + if(this.options.header === false){ + return; } - }catch(err){ - return [err]; - } - } - normalize_columns(columns){ - if(columns === undefined || columns === null){ - return []; - } - if(typeof columns !== 'object'){ - return [Error('Invalid option "columns": expect an array or an object')]; - } - if(!Array.isArray(columns)){ - const newcolumns = []; - for(const k in columns){ - newcolumns.push({ - key: k, - header: columns[k] - }); + if(this.options.columns === undefined){ + return; } - columns = newcolumns; - }else { - const newcolumns = []; - for(const column of columns){ - if(typeof column === 'string'){ - newcolumns.push({ - key: column, - header: column - }); - }else if(typeof column === 'object' && column !== undefined && !Array.isArray(column)){ - if(!column.key){ - return [Error('Invalid column definition: property "key" is required')]; - } - if(column.header === undefined){ - column.header = column.key; - } - newcolumns.push(column); + let err; + let headers = this.options.columns.map(column => column.header); + if(this.options.eof){ + [err, headers] = this.stringify(headers, true); + headers += this.options.record_delimiter; + }else { + [err, headers] = this.stringify(headers); + } + if(err) return err; + push(headers); + }, + __cast: function(value, context){ + const type = typeof value; + try{ + if(type === 'string'){ // Fine for 99% of the cases + return [undefined, this.options.cast.string(value, context)]; + }else if(type === 'bigint'){ + return [undefined, this.options.cast.bigint(value, context)]; + }else if(type === 'number'){ + return [undefined, this.options.cast.number(value, context)]; + }else if(type === 'boolean'){ + return [undefined, this.options.cast.boolean(value, context)]; + }else if(value instanceof Date){ + return [undefined, this.options.cast.date(value, context)]; + }else if(type === 'object' && value !== null){ + return [undefined, this.options.cast.object(value, context)]; }else { - return [Error('Invalid column definition: expect a string or an object')]; + return [undefined, value, value]; } + }catch(err){ + return [err]; } - columns = newcolumns; } - return [undefined, columns]; - } - } + }; + }; - const stringify = function(records, options={}){ + const stringify = function(records, opts={}){ const data = []; - const stringifier = new Stringifier(options); - stringifier.push = function(record){ - if(record === null){ - return; - } - data.push(record.toString()); + const [err, options] = normalize_options(opts); + if(err !== undefined) throw err; + const state = { + stop: false }; + // Information + const info = { + records: 0 + }; + const api = stringifier(options, state, info); + // stringifier.push = function(record){ + // if(record === null){ + // return; + // } + // data.push(record.toString()); + // }; for(const record of records){ - const err = stringifier.__transform(record, null); + const err = api.__transform(record, function(record){ + data.push(record); + }); if(err !== undefined) throw err; } return data.join(''); diff --git a/packages/csv-stringify/dist/umd/index.js b/packages/csv-stringify/dist/umd/index.js index 9691bcd43..35ac97df1 100644 --- a/packages/csv-stringify/dist/umd/index.js +++ b/packages/csv-stringify/dist/umd/index.js @@ -1,5576 +1,5599 @@ (function (global, factory) { - typeof exports === 'object' && typeof module !== 'undefined' ? factory(exports) : - typeof define === 'function' && define.amd ? define(['exports'], factory) : - (global = typeof globalThis !== 'undefined' ? globalThis : global || self, factory(global.csv_stringify = {})); + typeof exports === 'object' && typeof module !== 'undefined' ? factory(exports) : + typeof define === 'function' && define.amd ? define(['exports'], factory) : + (global = typeof globalThis !== 'undefined' ? globalThis : global || self, factory(global.csv_stringify = {})); })(this, (function (exports) { 'use strict'; - var global$1 = (typeof global !== "undefined" ? global : - typeof self !== "undefined" ? self : - typeof window !== "undefined" ? window : {}); - - var lookup = []; - var revLookup = []; - var Arr = typeof Uint8Array !== 'undefined' ? Uint8Array : Array; - var inited = false; - function init () { - inited = true; - var code = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'; - for (var i = 0, len = code.length; i < len; ++i) { - lookup[i] = code[i]; - revLookup[code.charCodeAt(i)] = i; - } - - revLookup['-'.charCodeAt(0)] = 62; - revLookup['_'.charCodeAt(0)] = 63; - } - - function toByteArray (b64) { - if (!inited) { - init(); - } - var i, j, l, tmp, placeHolders, arr; - var len = b64.length; - - if (len % 4 > 0) { - throw new Error('Invalid string. Length must be a multiple of 4') - } - - // the number of equal signs (place holders) - // if there are two placeholders, than the two characters before it - // represent one byte - // if there is only one, then the three characters before it represent 2 bytes - // this is just a cheap hack to not do indexOf twice - placeHolders = b64[len - 2] === '=' ? 2 : b64[len - 1] === '=' ? 1 : 0; - - // base64 is 4/3 + up to two characters of the original data - arr = new Arr(len * 3 / 4 - placeHolders); - - // if there are placeholders, only get up to the last complete 4 chars - l = placeHolders > 0 ? len - 4 : len; - - var L = 0; - - for (i = 0, j = 0; i < l; i += 4, j += 3) { - tmp = (revLookup[b64.charCodeAt(i)] << 18) | (revLookup[b64.charCodeAt(i + 1)] << 12) | (revLookup[b64.charCodeAt(i + 2)] << 6) | revLookup[b64.charCodeAt(i + 3)]; - arr[L++] = (tmp >> 16) & 0xFF; - arr[L++] = (tmp >> 8) & 0xFF; - arr[L++] = tmp & 0xFF; - } - - if (placeHolders === 2) { - tmp = (revLookup[b64.charCodeAt(i)] << 2) | (revLookup[b64.charCodeAt(i + 1)] >> 4); - arr[L++] = tmp & 0xFF; - } else if (placeHolders === 1) { - tmp = (revLookup[b64.charCodeAt(i)] << 10) | (revLookup[b64.charCodeAt(i + 1)] << 4) | (revLookup[b64.charCodeAt(i + 2)] >> 2); - arr[L++] = (tmp >> 8) & 0xFF; - arr[L++] = tmp & 0xFF; - } - - return arr - } - - function tripletToBase64 (num) { - return lookup[num >> 18 & 0x3F] + lookup[num >> 12 & 0x3F] + lookup[num >> 6 & 0x3F] + lookup[num & 0x3F] - } - - function encodeChunk (uint8, start, end) { - var tmp; - var output = []; - for (var i = start; i < end; i += 3) { - tmp = (uint8[i] << 16) + (uint8[i + 1] << 8) + (uint8[i + 2]); - output.push(tripletToBase64(tmp)); - } - return output.join('') - } - - function fromByteArray (uint8) { - if (!inited) { - init(); - } - var tmp; - var len = uint8.length; - var extraBytes = len % 3; // if we have 1 byte left, pad 2 bytes - var output = ''; - var parts = []; - var maxChunkLength = 16383; // must be multiple of 3 - - // go through the array every three bytes, we'll deal with trailing stuff later - for (var i = 0, len2 = len - extraBytes; i < len2; i += maxChunkLength) { - parts.push(encodeChunk(uint8, i, (i + maxChunkLength) > len2 ? len2 : (i + maxChunkLength))); - } - - // pad the end with zeros, but make sure to not forget the extra bytes - if (extraBytes === 1) { - tmp = uint8[len - 1]; - output += lookup[tmp >> 2]; - output += lookup[(tmp << 4) & 0x3F]; - output += '=='; - } else if (extraBytes === 2) { - tmp = (uint8[len - 2] << 8) + (uint8[len - 1]); - output += lookup[tmp >> 10]; - output += lookup[(tmp >> 4) & 0x3F]; - output += lookup[(tmp << 2) & 0x3F]; - output += '='; - } - - parts.push(output); - - return parts.join('') - } - - function read (buffer, offset, isLE, mLen, nBytes) { - var e, m; - var eLen = nBytes * 8 - mLen - 1; - var eMax = (1 << eLen) - 1; - var eBias = eMax >> 1; - var nBits = -7; - var i = isLE ? (nBytes - 1) : 0; - var d = isLE ? -1 : 1; - var s = buffer[offset + i]; - - i += d; - - e = s & ((1 << (-nBits)) - 1); - s >>= (-nBits); - nBits += eLen; - for (; nBits > 0; e = e * 256 + buffer[offset + i], i += d, nBits -= 8) {} - - m = e & ((1 << (-nBits)) - 1); - e >>= (-nBits); - nBits += mLen; - for (; nBits > 0; m = m * 256 + buffer[offset + i], i += d, nBits -= 8) {} - - if (e === 0) { - e = 1 - eBias; - } else if (e === eMax) { - return m ? NaN : ((s ? -1 : 1) * Infinity) - } else { - m = m + Math.pow(2, mLen); - e = e - eBias; - } - return (s ? -1 : 1) * m * Math.pow(2, e - mLen) - } - - function write (buffer, value, offset, isLE, mLen, nBytes) { - var e, m, c; - var eLen = nBytes * 8 - mLen - 1; - var eMax = (1 << eLen) - 1; - var eBias = eMax >> 1; - var rt = (mLen === 23 ? Math.pow(2, -24) - Math.pow(2, -77) : 0); - var i = isLE ? 0 : (nBytes - 1); - var d = isLE ? 1 : -1; - var s = value < 0 || (value === 0 && 1 / value < 0) ? 1 : 0; - - value = Math.abs(value); - - if (isNaN(value) || value === Infinity) { - m = isNaN(value) ? 1 : 0; - e = eMax; - } else { - e = Math.floor(Math.log(value) / Math.LN2); - if (value * (c = Math.pow(2, -e)) < 1) { - e--; - c *= 2; - } - if (e + eBias >= 1) { - value += rt / c; - } else { - value += rt * Math.pow(2, 1 - eBias); - } - if (value * c >= 2) { - e++; - c /= 2; - } - - if (e + eBias >= eMax) { - m = 0; - e = eMax; - } else if (e + eBias >= 1) { - m = (value * c - 1) * Math.pow(2, mLen); - e = e + eBias; - } else { - m = value * Math.pow(2, eBias - 1) * Math.pow(2, mLen); - e = 0; - } - } - - for (; mLen >= 8; buffer[offset + i] = m & 0xff, i += d, m /= 256, mLen -= 8) {} - - e = (e << mLen) | m; - eLen += mLen; - for (; eLen > 0; buffer[offset + i] = e & 0xff, i += d, e /= 256, eLen -= 8) {} - - buffer[offset + i - d] |= s * 128; - } - - var toString = {}.toString; - - var isArray$1 = Array.isArray || function (arr) { - return toString.call(arr) == '[object Array]'; - }; - - var INSPECT_MAX_BYTES = 50; - - /** - * If `Buffer.TYPED_ARRAY_SUPPORT`: - * === true Use Uint8Array implementation (fastest) - * === false Use Object implementation (most compatible, even IE6) - * - * Browsers that support typed arrays are IE 10+, Firefox 4+, Chrome 7+, Safari 5.1+, - * Opera 11.6+, iOS 4.2+. - * - * Due to various browser bugs, sometimes the Object implementation will be used even - * when the browser supports typed arrays. - * - * Note: - * - * - Firefox 4-29 lacks support for adding new properties to `Uint8Array` instances, - * See: https://bugzilla.mozilla.org/show_bug.cgi?id=695438. - * - * - Chrome 9-10 is missing the `TypedArray.prototype.subarray` function. - * - * - IE10 has a broken `TypedArray.prototype.subarray` function which returns arrays of - * incorrect length in some situations. - - * We detect these buggy browsers and set `Buffer.TYPED_ARRAY_SUPPORT` to `false` so they - * get the Object implementation, which is slower but behaves correctly. - */ - Buffer.TYPED_ARRAY_SUPPORT = global$1.TYPED_ARRAY_SUPPORT !== undefined - ? global$1.TYPED_ARRAY_SUPPORT - : true; - - function kMaxLength () { - return Buffer.TYPED_ARRAY_SUPPORT - ? 0x7fffffff - : 0x3fffffff - } - - function createBuffer (that, length) { - if (kMaxLength() < length) { - throw new RangeError('Invalid typed array length') - } - if (Buffer.TYPED_ARRAY_SUPPORT) { - // Return an augmented `Uint8Array` instance, for best performance - that = new Uint8Array(length); - that.__proto__ = Buffer.prototype; - } else { - // Fallback: Return an object instance of the Buffer class - if (that === null) { - that = new Buffer(length); - } - that.length = length; - } - - return that - } - - /** - * The Buffer constructor returns instances of `Uint8Array` that have their - * prototype changed to `Buffer.prototype`. Furthermore, `Buffer` is a subclass of - * `Uint8Array`, so the returned instances will have all the node `Buffer` methods - * and the `Uint8Array` methods. Square bracket notation works as expected -- it - * returns a single octet. - * - * The `Uint8Array` prototype remains unmodified. - */ - - function Buffer (arg, encodingOrOffset, length) { - if (!Buffer.TYPED_ARRAY_SUPPORT && !(this instanceof Buffer)) { - return new Buffer(arg, encodingOrOffset, length) - } - - // Common case. - if (typeof arg === 'number') { - if (typeof encodingOrOffset === 'string') { - throw new Error( - 'If encoding is specified then the first argument must be a string' - ) - } - return allocUnsafe(this, arg) - } - return from(this, arg, encodingOrOffset, length) - } - - Buffer.poolSize = 8192; // not used by this implementation - - // TODO: Legacy, not needed anymore. Remove in next major version. - Buffer._augment = function (arr) { - arr.__proto__ = Buffer.prototype; - return arr - }; - - function from (that, value, encodingOrOffset, length) { - if (typeof value === 'number') { - throw new TypeError('"value" argument must not be a number') - } - - if (typeof ArrayBuffer !== 'undefined' && value instanceof ArrayBuffer) { - return fromArrayBuffer(that, value, encodingOrOffset, length) - } - - if (typeof value === 'string') { - return fromString(that, value, encodingOrOffset) - } - - return fromObject(that, value) - } - - /** - * Functionally equivalent to Buffer(arg, encoding) but throws a TypeError - * if value is a number. - * Buffer.from(str[, encoding]) - * Buffer.from(array) - * Buffer.from(buffer) - * Buffer.from(arrayBuffer[, byteOffset[, length]]) - **/ - Buffer.from = function (value, encodingOrOffset, length) { - return from(null, value, encodingOrOffset, length) - }; - - if (Buffer.TYPED_ARRAY_SUPPORT) { - Buffer.prototype.__proto__ = Uint8Array.prototype; - Buffer.__proto__ = Uint8Array; - } - - function assertSize (size) { - if (typeof size !== 'number') { - throw new TypeError('"size" argument must be a number') - } else if (size < 0) { - throw new RangeError('"size" argument must not be negative') - } - } - - function alloc (that, size, fill, encoding) { - assertSize(size); - if (size <= 0) { - return createBuffer(that, size) - } - if (fill !== undefined) { - // Only pay attention to encoding if it's a string. This - // prevents accidentally sending in a number that would - // be interpretted as a start offset. - return typeof encoding === 'string' - ? createBuffer(that, size).fill(fill, encoding) - : createBuffer(that, size).fill(fill) - } - return createBuffer(that, size) - } - - /** - * Creates a new filled Buffer instance. - * alloc(size[, fill[, encoding]]) - **/ - Buffer.alloc = function (size, fill, encoding) { - return alloc(null, size, fill, encoding) - }; - - function allocUnsafe (that, size) { - assertSize(size); - that = createBuffer(that, size < 0 ? 0 : checked(size) | 0); - if (!Buffer.TYPED_ARRAY_SUPPORT) { - for (var i = 0; i < size; ++i) { - that[i] = 0; - } - } - return that - } - - /** - * Equivalent to Buffer(num), by default creates a non-zero-filled Buffer instance. - * */ - Buffer.allocUnsafe = function (size) { - return allocUnsafe(null, size) - }; - /** - * Equivalent to SlowBuffer(num), by default creates a non-zero-filled Buffer instance. - */ - Buffer.allocUnsafeSlow = function (size) { - return allocUnsafe(null, size) - }; - - function fromString (that, string, encoding) { - if (typeof encoding !== 'string' || encoding === '') { - encoding = 'utf8'; - } - - if (!Buffer.isEncoding(encoding)) { - throw new TypeError('"encoding" must be a valid string encoding') - } - - var length = byteLength(string, encoding) | 0; - that = createBuffer(that, length); - - var actual = that.write(string, encoding); - - if (actual !== length) { - // Writing a hex string, for example, that contains invalid characters will - // cause everything after the first invalid character to be ignored. (e.g. - // 'abxxcd' will be treated as 'ab') - that = that.slice(0, actual); - } - - return that - } - - function fromArrayLike (that, array) { - var length = array.length < 0 ? 0 : checked(array.length) | 0; - that = createBuffer(that, length); - for (var i = 0; i < length; i += 1) { - that[i] = array[i] & 255; - } - return that - } - - function fromArrayBuffer (that, array, byteOffset, length) { - array.byteLength; // this throws if `array` is not a valid ArrayBuffer - - if (byteOffset < 0 || array.byteLength < byteOffset) { - throw new RangeError('\'offset\' is out of bounds') - } - - if (array.byteLength < byteOffset + (length || 0)) { - throw new RangeError('\'length\' is out of bounds') - } - - if (byteOffset === undefined && length === undefined) { - array = new Uint8Array(array); - } else if (length === undefined) { - array = new Uint8Array(array, byteOffset); - } else { - array = new Uint8Array(array, byteOffset, length); - } - - if (Buffer.TYPED_ARRAY_SUPPORT) { - // Return an augmented `Uint8Array` instance, for best performance - that = array; - that.__proto__ = Buffer.prototype; - } else { - // Fallback: Return an object instance of the Buffer class - that = fromArrayLike(that, array); - } - return that - } - - function fromObject (that, obj) { - if (internalIsBuffer(obj)) { - var len = checked(obj.length) | 0; - that = createBuffer(that, len); - - if (that.length === 0) { - return that - } - - obj.copy(that, 0, 0, len); - return that - } - - if (obj) { - if ((typeof ArrayBuffer !== 'undefined' && - obj.buffer instanceof ArrayBuffer) || 'length' in obj) { - if (typeof obj.length !== 'number' || isnan(obj.length)) { - return createBuffer(that, 0) - } - return fromArrayLike(that, obj) - } - - if (obj.type === 'Buffer' && isArray$1(obj.data)) { - return fromArrayLike(that, obj.data) - } - } - - throw new TypeError('First argument must be a string, Buffer, ArrayBuffer, Array, or array-like object.') - } - - function checked (length) { - // Note: cannot use `length < kMaxLength()` here because that fails when - // length is NaN (which is otherwise coerced to zero.) - if (length >= kMaxLength()) { - throw new RangeError('Attempt to allocate Buffer larger than maximum ' + - 'size: 0x' + kMaxLength().toString(16) + ' bytes') - } - return length | 0 - } - Buffer.isBuffer = isBuffer; - function internalIsBuffer (b) { - return !!(b != null && b._isBuffer) - } - - Buffer.compare = function compare (a, b) { - if (!internalIsBuffer(a) || !internalIsBuffer(b)) { - throw new TypeError('Arguments must be Buffers') - } - - if (a === b) return 0 - - var x = a.length; - var y = b.length; - - for (var i = 0, len = Math.min(x, y); i < len; ++i) { - if (a[i] !== b[i]) { - x = a[i]; - y = b[i]; - break - } - } - - if (x < y) return -1 - if (y < x) return 1 - return 0 - }; - - Buffer.isEncoding = function isEncoding (encoding) { - switch (String(encoding).toLowerCase()) { - case 'hex': - case 'utf8': - case 'utf-8': - case 'ascii': - case 'latin1': - case 'binary': - case 'base64': - case 'ucs2': - case 'ucs-2': - case 'utf16le': - case 'utf-16le': - return true - default: - return false - } - }; - - Buffer.concat = function concat (list, length) { - if (!isArray$1(list)) { - throw new TypeError('"list" argument must be an Array of Buffers') - } - - if (list.length === 0) { - return Buffer.alloc(0) - } - - var i; - if (length === undefined) { - length = 0; - for (i = 0; i < list.length; ++i) { - length += list[i].length; - } - } - - var buffer = Buffer.allocUnsafe(length); - var pos = 0; - for (i = 0; i < list.length; ++i) { - var buf = list[i]; - if (!internalIsBuffer(buf)) { - throw new TypeError('"list" argument must be an Array of Buffers') - } - buf.copy(buffer, pos); - pos += buf.length; - } - return buffer - }; - - function byteLength (string, encoding) { - if (internalIsBuffer(string)) { - return string.length - } - if (typeof ArrayBuffer !== 'undefined' && typeof ArrayBuffer.isView === 'function' && - (ArrayBuffer.isView(string) || string instanceof ArrayBuffer)) { - return string.byteLength - } - if (typeof string !== 'string') { - string = '' + string; - } - - var len = string.length; - if (len === 0) return 0 - - // Use a for loop to avoid recursion - var loweredCase = false; - for (;;) { - switch (encoding) { - case 'ascii': - case 'latin1': - case 'binary': - return len - case 'utf8': - case 'utf-8': - case undefined: - return utf8ToBytes(string).length - case 'ucs2': - case 'ucs-2': - case 'utf16le': - case 'utf-16le': - return len * 2 - case 'hex': - return len >>> 1 - case 'base64': - return base64ToBytes(string).length - default: - if (loweredCase) return utf8ToBytes(string).length // assume utf8 - encoding = ('' + encoding).toLowerCase(); - loweredCase = true; - } - } - } - Buffer.byteLength = byteLength; - - function slowToString (encoding, start, end) { - var loweredCase = false; - - // No need to verify that "this.length <= MAX_UINT32" since it's a read-only - // property of a typed array. - - // This behaves neither like String nor Uint8Array in that we set start/end - // to their upper/lower bounds if the value passed is out of range. - // undefined is handled specially as per ECMA-262 6th Edition, - // Section 13.3.3.7 Runtime Semantics: KeyedBindingInitialization. - if (start === undefined || start < 0) { - start = 0; - } - // Return early if start > this.length. Done here to prevent potential uint32 - // coercion fail below. - if (start > this.length) { - return '' - } - - if (end === undefined || end > this.length) { - end = this.length; - } - - if (end <= 0) { - return '' - } - - // Force coersion to uint32. This will also coerce falsey/NaN values to 0. - end >>>= 0; - start >>>= 0; - - if (end <= start) { - return '' - } - - if (!encoding) encoding = 'utf8'; - - while (true) { - switch (encoding) { - case 'hex': - return hexSlice(this, start, end) - - case 'utf8': - case 'utf-8': - return utf8Slice(this, start, end) - - case 'ascii': - return asciiSlice(this, start, end) - - case 'latin1': - case 'binary': - return latin1Slice(this, start, end) - - case 'base64': - return base64Slice(this, start, end) - - case 'ucs2': - case 'ucs-2': - case 'utf16le': - case 'utf-16le': - return utf16leSlice(this, start, end) - - default: - if (loweredCase) throw new TypeError('Unknown encoding: ' + encoding) - encoding = (encoding + '').toLowerCase(); - loweredCase = true; - } - } - } - - // The property is used by `Buffer.isBuffer` and `is-buffer` (in Safari 5-7) to detect - // Buffer instances. - Buffer.prototype._isBuffer = true; - - function swap (b, n, m) { - var i = b[n]; - b[n] = b[m]; - b[m] = i; - } - - Buffer.prototype.swap16 = function swap16 () { - var len = this.length; - if (len % 2 !== 0) { - throw new RangeError('Buffer size must be a multiple of 16-bits') - } - for (var i = 0; i < len; i += 2) { - swap(this, i, i + 1); - } - return this - }; - - Buffer.prototype.swap32 = function swap32 () { - var len = this.length; - if (len % 4 !== 0) { - throw new RangeError('Buffer size must be a multiple of 32-bits') - } - for (var i = 0; i < len; i += 4) { - swap(this, i, i + 3); - swap(this, i + 1, i + 2); - } - return this - }; - - Buffer.prototype.swap64 = function swap64 () { - var len = this.length; - if (len % 8 !== 0) { - throw new RangeError('Buffer size must be a multiple of 64-bits') - } - for (var i = 0; i < len; i += 8) { - swap(this, i, i + 7); - swap(this, i + 1, i + 6); - swap(this, i + 2, i + 5); - swap(this, i + 3, i + 4); - } - return this - }; - - Buffer.prototype.toString = function toString () { - var length = this.length | 0; - if (length === 0) return '' - if (arguments.length === 0) return utf8Slice(this, 0, length) - return slowToString.apply(this, arguments) - }; - - Buffer.prototype.equals = function equals (b) { - if (!internalIsBuffer(b)) throw new TypeError('Argument must be a Buffer') - if (this === b) return true - return Buffer.compare(this, b) === 0 - }; - - Buffer.prototype.inspect = function inspect () { - var str = ''; - var max = INSPECT_MAX_BYTES; - if (this.length > 0) { - str = this.toString('hex', 0, max).match(/.{2}/g).join(' '); - if (this.length > max) str += ' ... '; - } - return '' - }; - - Buffer.prototype.compare = function compare (target, start, end, thisStart, thisEnd) { - if (!internalIsBuffer(target)) { - throw new TypeError('Argument must be a Buffer') - } - - if (start === undefined) { - start = 0; - } - if (end === undefined) { - end = target ? target.length : 0; - } - if (thisStart === undefined) { - thisStart = 0; - } - if (thisEnd === undefined) { - thisEnd = this.length; - } - - if (start < 0 || end > target.length || thisStart < 0 || thisEnd > this.length) { - throw new RangeError('out of range index') - } - - if (thisStart >= thisEnd && start >= end) { - return 0 - } - if (thisStart >= thisEnd) { - return -1 - } - if (start >= end) { - return 1 - } - - start >>>= 0; - end >>>= 0; - thisStart >>>= 0; - thisEnd >>>= 0; - - if (this === target) return 0 - - var x = thisEnd - thisStart; - var y = end - start; - var len = Math.min(x, y); - - var thisCopy = this.slice(thisStart, thisEnd); - var targetCopy = target.slice(start, end); - - for (var i = 0; i < len; ++i) { - if (thisCopy[i] !== targetCopy[i]) { - x = thisCopy[i]; - y = targetCopy[i]; - break - } - } - - if (x < y) return -1 - if (y < x) return 1 - return 0 - }; - - // Finds either the first index of `val` in `buffer` at offset >= `byteOffset`, - // OR the last index of `val` in `buffer` at offset <= `byteOffset`. - // - // Arguments: - // - buffer - a Buffer to search - // - val - a string, Buffer, or number - // - byteOffset - an index into `buffer`; will be clamped to an int32 - // - encoding - an optional encoding, relevant is val is a string - // - dir - true for indexOf, false for lastIndexOf - function bidirectionalIndexOf (buffer, val, byteOffset, encoding, dir) { - // Empty buffer means no match - if (buffer.length === 0) return -1 - - // Normalize byteOffset - if (typeof byteOffset === 'string') { - encoding = byteOffset; - byteOffset = 0; - } else if (byteOffset > 0x7fffffff) { - byteOffset = 0x7fffffff; - } else if (byteOffset < -0x80000000) { - byteOffset = -0x80000000; - } - byteOffset = +byteOffset; // Coerce to Number. - if (isNaN(byteOffset)) { - // byteOffset: it it's undefined, null, NaN, "foo", etc, search whole buffer - byteOffset = dir ? 0 : (buffer.length - 1); - } - - // Normalize byteOffset: negative offsets start from the end of the buffer - if (byteOffset < 0) byteOffset = buffer.length + byteOffset; - if (byteOffset >= buffer.length) { - if (dir) return -1 - else byteOffset = buffer.length - 1; - } else if (byteOffset < 0) { - if (dir) byteOffset = 0; - else return -1 - } - - // Normalize val - if (typeof val === 'string') { - val = Buffer.from(val, encoding); - } - - // Finally, search either indexOf (if dir is true) or lastIndexOf - if (internalIsBuffer(val)) { - // Special case: looking for empty string/buffer always fails - if (val.length === 0) { - return -1 - } - return arrayIndexOf(buffer, val, byteOffset, encoding, dir) - } else if (typeof val === 'number') { - val = val & 0xFF; // Search for a byte value [0-255] - if (Buffer.TYPED_ARRAY_SUPPORT && - typeof Uint8Array.prototype.indexOf === 'function') { - if (dir) { - return Uint8Array.prototype.indexOf.call(buffer, val, byteOffset) - } else { - return Uint8Array.prototype.lastIndexOf.call(buffer, val, byteOffset) - } - } - return arrayIndexOf(buffer, [ val ], byteOffset, encoding, dir) - } - - throw new TypeError('val must be string, number or Buffer') - } - - function arrayIndexOf (arr, val, byteOffset, encoding, dir) { - var indexSize = 1; - var arrLength = arr.length; - var valLength = val.length; - - if (encoding !== undefined) { - encoding = String(encoding).toLowerCase(); - if (encoding === 'ucs2' || encoding === 'ucs-2' || - encoding === 'utf16le' || encoding === 'utf-16le') { - if (arr.length < 2 || val.length < 2) { - return -1 - } - indexSize = 2; - arrLength /= 2; - valLength /= 2; - byteOffset /= 2; - } - } - - function read (buf, i) { - if (indexSize === 1) { - return buf[i] - } else { - return buf.readUInt16BE(i * indexSize) - } - } - - var i; - if (dir) { - var foundIndex = -1; - for (i = byteOffset; i < arrLength; i++) { - if (read(arr, i) === read(val, foundIndex === -1 ? 0 : i - foundIndex)) { - if (foundIndex === -1) foundIndex = i; - if (i - foundIndex + 1 === valLength) return foundIndex * indexSize - } else { - if (foundIndex !== -1) i -= i - foundIndex; - foundIndex = -1; - } - } - } else { - if (byteOffset + valLength > arrLength) byteOffset = arrLength - valLength; - for (i = byteOffset; i >= 0; i--) { - var found = true; - for (var j = 0; j < valLength; j++) { - if (read(arr, i + j) !== read(val, j)) { - found = false; - break - } - } - if (found) return i - } - } - - return -1 - } - - Buffer.prototype.includes = function includes (val, byteOffset, encoding) { - return this.indexOf(val, byteOffset, encoding) !== -1 - }; - - Buffer.prototype.indexOf = function indexOf (val, byteOffset, encoding) { - return bidirectionalIndexOf(this, val, byteOffset, encoding, true) - }; - - Buffer.prototype.lastIndexOf = function lastIndexOf (val, byteOffset, encoding) { - return bidirectionalIndexOf(this, val, byteOffset, encoding, false) - }; - - function hexWrite (buf, string, offset, length) { - offset = Number(offset) || 0; - var remaining = buf.length - offset; - if (!length) { - length = remaining; - } else { - length = Number(length); - if (length > remaining) { - length = remaining; - } - } - - // must be an even number of digits - var strLen = string.length; - if (strLen % 2 !== 0) throw new TypeError('Invalid hex string') - - if (length > strLen / 2) { - length = strLen / 2; - } - for (var i = 0; i < length; ++i) { - var parsed = parseInt(string.substr(i * 2, 2), 16); - if (isNaN(parsed)) return i - buf[offset + i] = parsed; - } - return i - } - - function utf8Write (buf, string, offset, length) { - return blitBuffer(utf8ToBytes(string, buf.length - offset), buf, offset, length) - } - - function asciiWrite (buf, string, offset, length) { - return blitBuffer(asciiToBytes(string), buf, offset, length) - } - - function latin1Write (buf, string, offset, length) { - return asciiWrite(buf, string, offset, length) - } - - function base64Write (buf, string, offset, length) { - return blitBuffer(base64ToBytes(string), buf, offset, length) - } - - function ucs2Write (buf, string, offset, length) { - return blitBuffer(utf16leToBytes(string, buf.length - offset), buf, offset, length) - } - - Buffer.prototype.write = function write (string, offset, length, encoding) { - // Buffer#write(string) - if (offset === undefined) { - encoding = 'utf8'; - length = this.length; - offset = 0; - // Buffer#write(string, encoding) - } else if (length === undefined && typeof offset === 'string') { - encoding = offset; - length = this.length; - offset = 0; - // Buffer#write(string, offset[, length][, encoding]) - } else if (isFinite(offset)) { - offset = offset | 0; - if (isFinite(length)) { - length = length | 0; - if (encoding === undefined) encoding = 'utf8'; - } else { - encoding = length; - length = undefined; - } - // legacy write(string, encoding, offset, length) - remove in v0.13 - } else { - throw new Error( - 'Buffer.write(string, encoding, offset[, length]) is no longer supported' - ) - } - - var remaining = this.length - offset; - if (length === undefined || length > remaining) length = remaining; - - if ((string.length > 0 && (length < 0 || offset < 0)) || offset > this.length) { - throw new RangeError('Attempt to write outside buffer bounds') - } - - if (!encoding) encoding = 'utf8'; - - var loweredCase = false; - for (;;) { - switch (encoding) { - case 'hex': - return hexWrite(this, string, offset, length) - - case 'utf8': - case 'utf-8': - return utf8Write(this, string, offset, length) - - case 'ascii': - return asciiWrite(this, string, offset, length) - - case 'latin1': - case 'binary': - return latin1Write(this, string, offset, length) - - case 'base64': - // Warning: maxLength not taken into account in base64Write - return base64Write(this, string, offset, length) - - case 'ucs2': - case 'ucs-2': - case 'utf16le': - case 'utf-16le': - return ucs2Write(this, string, offset, length) - - default: - if (loweredCase) throw new TypeError('Unknown encoding: ' + encoding) - encoding = ('' + encoding).toLowerCase(); - loweredCase = true; - } - } - }; - - Buffer.prototype.toJSON = function toJSON () { - return { - type: 'Buffer', - data: Array.prototype.slice.call(this._arr || this, 0) - } - }; - - function base64Slice (buf, start, end) { - if (start === 0 && end === buf.length) { - return fromByteArray(buf) - } else { - return fromByteArray(buf.slice(start, end)) - } - } - - function utf8Slice (buf, start, end) { - end = Math.min(buf.length, end); - var res = []; - - var i = start; - while (i < end) { - var firstByte = buf[i]; - var codePoint = null; - var bytesPerSequence = (firstByte > 0xEF) ? 4 - : (firstByte > 0xDF) ? 3 - : (firstByte > 0xBF) ? 2 - : 1; - - if (i + bytesPerSequence <= end) { - var secondByte, thirdByte, fourthByte, tempCodePoint; - - switch (bytesPerSequence) { - case 1: - if (firstByte < 0x80) { - codePoint = firstByte; - } - break - case 2: - secondByte = buf[i + 1]; - if ((secondByte & 0xC0) === 0x80) { - tempCodePoint = (firstByte & 0x1F) << 0x6 | (secondByte & 0x3F); - if (tempCodePoint > 0x7F) { - codePoint = tempCodePoint; - } - } - break - case 3: - secondByte = buf[i + 1]; - thirdByte = buf[i + 2]; - if ((secondByte & 0xC0) === 0x80 && (thirdByte & 0xC0) === 0x80) { - tempCodePoint = (firstByte & 0xF) << 0xC | (secondByte & 0x3F) << 0x6 | (thirdByte & 0x3F); - if (tempCodePoint > 0x7FF && (tempCodePoint < 0xD800 || tempCodePoint > 0xDFFF)) { - codePoint = tempCodePoint; - } - } - break - case 4: - secondByte = buf[i + 1]; - thirdByte = buf[i + 2]; - fourthByte = buf[i + 3]; - if ((secondByte & 0xC0) === 0x80 && (thirdByte & 0xC0) === 0x80 && (fourthByte & 0xC0) === 0x80) { - tempCodePoint = (firstByte & 0xF) << 0x12 | (secondByte & 0x3F) << 0xC | (thirdByte & 0x3F) << 0x6 | (fourthByte & 0x3F); - if (tempCodePoint > 0xFFFF && tempCodePoint < 0x110000) { - codePoint = tempCodePoint; - } - } - } - } - - if (codePoint === null) { - // we did not generate a valid codePoint so insert a - // replacement char (U+FFFD) and advance only 1 byte - codePoint = 0xFFFD; - bytesPerSequence = 1; - } else if (codePoint > 0xFFFF) { - // encode to utf16 (surrogate pair dance) - codePoint -= 0x10000; - res.push(codePoint >>> 10 & 0x3FF | 0xD800); - codePoint = 0xDC00 | codePoint & 0x3FF; - } - - res.push(codePoint); - i += bytesPerSequence; - } - - return decodeCodePointsArray(res) - } - - // Based on http://stackoverflow.com/a/22747272/680742, the browser with - // the lowest limit is Chrome, with 0x10000 args. - // We go 1 magnitude less, for safety - var MAX_ARGUMENTS_LENGTH = 0x1000; - - function decodeCodePointsArray (codePoints) { - var len = codePoints.length; - if (len <= MAX_ARGUMENTS_LENGTH) { - return String.fromCharCode.apply(String, codePoints) // avoid extra slice() - } - - // Decode in chunks to avoid "call stack size exceeded". - var res = ''; - var i = 0; - while (i < len) { - res += String.fromCharCode.apply( - String, - codePoints.slice(i, i += MAX_ARGUMENTS_LENGTH) - ); - } - return res - } - - function asciiSlice (buf, start, end) { - var ret = ''; - end = Math.min(buf.length, end); - - for (var i = start; i < end; ++i) { - ret += String.fromCharCode(buf[i] & 0x7F); - } - return ret - } - - function latin1Slice (buf, start, end) { - var ret = ''; - end = Math.min(buf.length, end); - - for (var i = start; i < end; ++i) { - ret += String.fromCharCode(buf[i]); - } - return ret - } - - function hexSlice (buf, start, end) { - var len = buf.length; - - if (!start || start < 0) start = 0; - if (!end || end < 0 || end > len) end = len; - - var out = ''; - for (var i = start; i < end; ++i) { - out += toHex(buf[i]); - } - return out - } - - function utf16leSlice (buf, start, end) { - var bytes = buf.slice(start, end); - var res = ''; - for (var i = 0; i < bytes.length; i += 2) { - res += String.fromCharCode(bytes[i] + bytes[i + 1] * 256); - } - return res - } - - Buffer.prototype.slice = function slice (start, end) { - var len = this.length; - start = ~~start; - end = end === undefined ? len : ~~end; - - if (start < 0) { - start += len; - if (start < 0) start = 0; - } else if (start > len) { - start = len; - } - - if (end < 0) { - end += len; - if (end < 0) end = 0; - } else if (end > len) { - end = len; - } - - if (end < start) end = start; - - var newBuf; - if (Buffer.TYPED_ARRAY_SUPPORT) { - newBuf = this.subarray(start, end); - newBuf.__proto__ = Buffer.prototype; - } else { - var sliceLen = end - start; - newBuf = new Buffer(sliceLen, undefined); - for (var i = 0; i < sliceLen; ++i) { - newBuf[i] = this[i + start]; - } - } - - return newBuf - }; - - /* - * Need to make sure that buffer isn't trying to write out of bounds. - */ - function checkOffset (offset, ext, length) { - if ((offset % 1) !== 0 || offset < 0) throw new RangeError('offset is not uint') - if (offset + ext > length) throw new RangeError('Trying to access beyond buffer length') - } - - Buffer.prototype.readUIntLE = function readUIntLE (offset, byteLength, noAssert) { - offset = offset | 0; - byteLength = byteLength | 0; - if (!noAssert) checkOffset(offset, byteLength, this.length); - - var val = this[offset]; - var mul = 1; - var i = 0; - while (++i < byteLength && (mul *= 0x100)) { - val += this[offset + i] * mul; - } - - return val - }; - - Buffer.prototype.readUIntBE = function readUIntBE (offset, byteLength, noAssert) { - offset = offset | 0; - byteLength = byteLength | 0; - if (!noAssert) { - checkOffset(offset, byteLength, this.length); - } - - var val = this[offset + --byteLength]; - var mul = 1; - while (byteLength > 0 && (mul *= 0x100)) { - val += this[offset + --byteLength] * mul; - } - - return val - }; - - Buffer.prototype.readUInt8 = function readUInt8 (offset, noAssert) { - if (!noAssert) checkOffset(offset, 1, this.length); - return this[offset] - }; - - Buffer.prototype.readUInt16LE = function readUInt16LE (offset, noAssert) { - if (!noAssert) checkOffset(offset, 2, this.length); - return this[offset] | (this[offset + 1] << 8) - }; - - Buffer.prototype.readUInt16BE = function readUInt16BE (offset, noAssert) { - if (!noAssert) checkOffset(offset, 2, this.length); - return (this[offset] << 8) | this[offset + 1] - }; - - Buffer.prototype.readUInt32LE = function readUInt32LE (offset, noAssert) { - if (!noAssert) checkOffset(offset, 4, this.length); - - return ((this[offset]) | - (this[offset + 1] << 8) | - (this[offset + 2] << 16)) + - (this[offset + 3] * 0x1000000) - }; - - Buffer.prototype.readUInt32BE = function readUInt32BE (offset, noAssert) { - if (!noAssert) checkOffset(offset, 4, this.length); - - return (this[offset] * 0x1000000) + - ((this[offset + 1] << 16) | - (this[offset + 2] << 8) | - this[offset + 3]) - }; - - Buffer.prototype.readIntLE = function readIntLE (offset, byteLength, noAssert) { - offset = offset | 0; - byteLength = byteLength | 0; - if (!noAssert) checkOffset(offset, byteLength, this.length); - - var val = this[offset]; - var mul = 1; - var i = 0; - while (++i < byteLength && (mul *= 0x100)) { - val += this[offset + i] * mul; - } - mul *= 0x80; - - if (val >= mul) val -= Math.pow(2, 8 * byteLength); - - return val - }; - - Buffer.prototype.readIntBE = function readIntBE (offset, byteLength, noAssert) { - offset = offset | 0; - byteLength = byteLength | 0; - if (!noAssert) checkOffset(offset, byteLength, this.length); - - var i = byteLength; - var mul = 1; - var val = this[offset + --i]; - while (i > 0 && (mul *= 0x100)) { - val += this[offset + --i] * mul; - } - mul *= 0x80; - - if (val >= mul) val -= Math.pow(2, 8 * byteLength); - - return val - }; - - Buffer.prototype.readInt8 = function readInt8 (offset, noAssert) { - if (!noAssert) checkOffset(offset, 1, this.length); - if (!(this[offset] & 0x80)) return (this[offset]) - return ((0xff - this[offset] + 1) * -1) - }; - - Buffer.prototype.readInt16LE = function readInt16LE (offset, noAssert) { - if (!noAssert) checkOffset(offset, 2, this.length); - var val = this[offset] | (this[offset + 1] << 8); - return (val & 0x8000) ? val | 0xFFFF0000 : val - }; - - Buffer.prototype.readInt16BE = function readInt16BE (offset, noAssert) { - if (!noAssert) checkOffset(offset, 2, this.length); - var val = this[offset + 1] | (this[offset] << 8); - return (val & 0x8000) ? val | 0xFFFF0000 : val - }; - - Buffer.prototype.readInt32LE = function readInt32LE (offset, noAssert) { - if (!noAssert) checkOffset(offset, 4, this.length); - - return (this[offset]) | - (this[offset + 1] << 8) | - (this[offset + 2] << 16) | - (this[offset + 3] << 24) - }; - - Buffer.prototype.readInt32BE = function readInt32BE (offset, noAssert) { - if (!noAssert) checkOffset(offset, 4, this.length); - - return (this[offset] << 24) | - (this[offset + 1] << 16) | - (this[offset + 2] << 8) | - (this[offset + 3]) - }; - - Buffer.prototype.readFloatLE = function readFloatLE (offset, noAssert) { - if (!noAssert) checkOffset(offset, 4, this.length); - return read(this, offset, true, 23, 4) - }; - - Buffer.prototype.readFloatBE = function readFloatBE (offset, noAssert) { - if (!noAssert) checkOffset(offset, 4, this.length); - return read(this, offset, false, 23, 4) - }; - - Buffer.prototype.readDoubleLE = function readDoubleLE (offset, noAssert) { - if (!noAssert) checkOffset(offset, 8, this.length); - return read(this, offset, true, 52, 8) - }; - - Buffer.prototype.readDoubleBE = function readDoubleBE (offset, noAssert) { - if (!noAssert) checkOffset(offset, 8, this.length); - return read(this, offset, false, 52, 8) - }; - - function checkInt (buf, value, offset, ext, max, min) { - if (!internalIsBuffer(buf)) throw new TypeError('"buffer" argument must be a Buffer instance') - if (value > max || value < min) throw new RangeError('"value" argument is out of bounds') - if (offset + ext > buf.length) throw new RangeError('Index out of range') - } - - Buffer.prototype.writeUIntLE = function writeUIntLE (value, offset, byteLength, noAssert) { - value = +value; - offset = offset | 0; - byteLength = byteLength | 0; - if (!noAssert) { - var maxBytes = Math.pow(2, 8 * byteLength) - 1; - checkInt(this, value, offset, byteLength, maxBytes, 0); - } - - var mul = 1; - var i = 0; - this[offset] = value & 0xFF; - while (++i < byteLength && (mul *= 0x100)) { - this[offset + i] = (value / mul) & 0xFF; - } - - return offset + byteLength - }; - - Buffer.prototype.writeUIntBE = function writeUIntBE (value, offset, byteLength, noAssert) { - value = +value; - offset = offset | 0; - byteLength = byteLength | 0; - if (!noAssert) { - var maxBytes = Math.pow(2, 8 * byteLength) - 1; - checkInt(this, value, offset, byteLength, maxBytes, 0); - } - - var i = byteLength - 1; - var mul = 1; - this[offset + i] = value & 0xFF; - while (--i >= 0 && (mul *= 0x100)) { - this[offset + i] = (value / mul) & 0xFF; - } - - return offset + byteLength - }; - - Buffer.prototype.writeUInt8 = function writeUInt8 (value, offset, noAssert) { - value = +value; - offset = offset | 0; - if (!noAssert) checkInt(this, value, offset, 1, 0xff, 0); - if (!Buffer.TYPED_ARRAY_SUPPORT) value = Math.floor(value); - this[offset] = (value & 0xff); - return offset + 1 - }; - - function objectWriteUInt16 (buf, value, offset, littleEndian) { - if (value < 0) value = 0xffff + value + 1; - for (var i = 0, j = Math.min(buf.length - offset, 2); i < j; ++i) { - buf[offset + i] = (value & (0xff << (8 * (littleEndian ? i : 1 - i)))) >>> - (littleEndian ? i : 1 - i) * 8; - } - } - - Buffer.prototype.writeUInt16LE = function writeUInt16LE (value, offset, noAssert) { - value = +value; - offset = offset | 0; - if (!noAssert) checkInt(this, value, offset, 2, 0xffff, 0); - if (Buffer.TYPED_ARRAY_SUPPORT) { - this[offset] = (value & 0xff); - this[offset + 1] = (value >>> 8); - } else { - objectWriteUInt16(this, value, offset, true); - } - return offset + 2 - }; - - Buffer.prototype.writeUInt16BE = function writeUInt16BE (value, offset, noAssert) { - value = +value; - offset = offset | 0; - if (!noAssert) checkInt(this, value, offset, 2, 0xffff, 0); - if (Buffer.TYPED_ARRAY_SUPPORT) { - this[offset] = (value >>> 8); - this[offset + 1] = (value & 0xff); - } else { - objectWriteUInt16(this, value, offset, false); - } - return offset + 2 - }; - - function objectWriteUInt32 (buf, value, offset, littleEndian) { - if (value < 0) value = 0xffffffff + value + 1; - for (var i = 0, j = Math.min(buf.length - offset, 4); i < j; ++i) { - buf[offset + i] = (value >>> (littleEndian ? i : 3 - i) * 8) & 0xff; - } - } - - Buffer.prototype.writeUInt32LE = function writeUInt32LE (value, offset, noAssert) { - value = +value; - offset = offset | 0; - if (!noAssert) checkInt(this, value, offset, 4, 0xffffffff, 0); - if (Buffer.TYPED_ARRAY_SUPPORT) { - this[offset + 3] = (value >>> 24); - this[offset + 2] = (value >>> 16); - this[offset + 1] = (value >>> 8); - this[offset] = (value & 0xff); - } else { - objectWriteUInt32(this, value, offset, true); - } - return offset + 4 - }; - - Buffer.prototype.writeUInt32BE = function writeUInt32BE (value, offset, noAssert) { - value = +value; - offset = offset | 0; - if (!noAssert) checkInt(this, value, offset, 4, 0xffffffff, 0); - if (Buffer.TYPED_ARRAY_SUPPORT) { - this[offset] = (value >>> 24); - this[offset + 1] = (value >>> 16); - this[offset + 2] = (value >>> 8); - this[offset + 3] = (value & 0xff); - } else { - objectWriteUInt32(this, value, offset, false); - } - return offset + 4 - }; - - Buffer.prototype.writeIntLE = function writeIntLE (value, offset, byteLength, noAssert) { - value = +value; - offset = offset | 0; - if (!noAssert) { - var limit = Math.pow(2, 8 * byteLength - 1); - - checkInt(this, value, offset, byteLength, limit - 1, -limit); - } - - var i = 0; - var mul = 1; - var sub = 0; - this[offset] = value & 0xFF; - while (++i < byteLength && (mul *= 0x100)) { - if (value < 0 && sub === 0 && this[offset + i - 1] !== 0) { - sub = 1; - } - this[offset + i] = ((value / mul) >> 0) - sub & 0xFF; - } - - return offset + byteLength - }; - - Buffer.prototype.writeIntBE = function writeIntBE (value, offset, byteLength, noAssert) { - value = +value; - offset = offset | 0; - if (!noAssert) { - var limit = Math.pow(2, 8 * byteLength - 1); - - checkInt(this, value, offset, byteLength, limit - 1, -limit); - } - - var i = byteLength - 1; - var mul = 1; - var sub = 0; - this[offset + i] = value & 0xFF; - while (--i >= 0 && (mul *= 0x100)) { - if (value < 0 && sub === 0 && this[offset + i + 1] !== 0) { - sub = 1; - } - this[offset + i] = ((value / mul) >> 0) - sub & 0xFF; - } - - return offset + byteLength - }; - - Buffer.prototype.writeInt8 = function writeInt8 (value, offset, noAssert) { - value = +value; - offset = offset | 0; - if (!noAssert) checkInt(this, value, offset, 1, 0x7f, -0x80); - if (!Buffer.TYPED_ARRAY_SUPPORT) value = Math.floor(value); - if (value < 0) value = 0xff + value + 1; - this[offset] = (value & 0xff); - return offset + 1 - }; - - Buffer.prototype.writeInt16LE = function writeInt16LE (value, offset, noAssert) { - value = +value; - offset = offset | 0; - if (!noAssert) checkInt(this, value, offset, 2, 0x7fff, -0x8000); - if (Buffer.TYPED_ARRAY_SUPPORT) { - this[offset] = (value & 0xff); - this[offset + 1] = (value >>> 8); - } else { - objectWriteUInt16(this, value, offset, true); - } - return offset + 2 - }; - - Buffer.prototype.writeInt16BE = function writeInt16BE (value, offset, noAssert) { - value = +value; - offset = offset | 0; - if (!noAssert) checkInt(this, value, offset, 2, 0x7fff, -0x8000); - if (Buffer.TYPED_ARRAY_SUPPORT) { - this[offset] = (value >>> 8); - this[offset + 1] = (value & 0xff); - } else { - objectWriteUInt16(this, value, offset, false); - } - return offset + 2 - }; - - Buffer.prototype.writeInt32LE = function writeInt32LE (value, offset, noAssert) { - value = +value; - offset = offset | 0; - if (!noAssert) checkInt(this, value, offset, 4, 0x7fffffff, -0x80000000); - if (Buffer.TYPED_ARRAY_SUPPORT) { - this[offset] = (value & 0xff); - this[offset + 1] = (value >>> 8); - this[offset + 2] = (value >>> 16); - this[offset + 3] = (value >>> 24); - } else { - objectWriteUInt32(this, value, offset, true); - } - return offset + 4 - }; - - Buffer.prototype.writeInt32BE = function writeInt32BE (value, offset, noAssert) { - value = +value; - offset = offset | 0; - if (!noAssert) checkInt(this, value, offset, 4, 0x7fffffff, -0x80000000); - if (value < 0) value = 0xffffffff + value + 1; - if (Buffer.TYPED_ARRAY_SUPPORT) { - this[offset] = (value >>> 24); - this[offset + 1] = (value >>> 16); - this[offset + 2] = (value >>> 8); - this[offset + 3] = (value & 0xff); - } else { - objectWriteUInt32(this, value, offset, false); - } - return offset + 4 - }; - - function checkIEEE754 (buf, value, offset, ext, max, min) { - if (offset + ext > buf.length) throw new RangeError('Index out of range') - if (offset < 0) throw new RangeError('Index out of range') - } - - function writeFloat (buf, value, offset, littleEndian, noAssert) { - if (!noAssert) { - checkIEEE754(buf, value, offset, 4); - } - write(buf, value, offset, littleEndian, 23, 4); - return offset + 4 - } - - Buffer.prototype.writeFloatLE = function writeFloatLE (value, offset, noAssert) { - return writeFloat(this, value, offset, true, noAssert) - }; - - Buffer.prototype.writeFloatBE = function writeFloatBE (value, offset, noAssert) { - return writeFloat(this, value, offset, false, noAssert) - }; - - function writeDouble (buf, value, offset, littleEndian, noAssert) { - if (!noAssert) { - checkIEEE754(buf, value, offset, 8); - } - write(buf, value, offset, littleEndian, 52, 8); - return offset + 8 - } - - Buffer.prototype.writeDoubleLE = function writeDoubleLE (value, offset, noAssert) { - return writeDouble(this, value, offset, true, noAssert) - }; - - Buffer.prototype.writeDoubleBE = function writeDoubleBE (value, offset, noAssert) { - return writeDouble(this, value, offset, false, noAssert) - }; - - // copy(targetBuffer, targetStart=0, sourceStart=0, sourceEnd=buffer.length) - Buffer.prototype.copy = function copy (target, targetStart, start, end) { - if (!start) start = 0; - if (!end && end !== 0) end = this.length; - if (targetStart >= target.length) targetStart = target.length; - if (!targetStart) targetStart = 0; - if (end > 0 && end < start) end = start; - - // Copy 0 bytes; we're done - if (end === start) return 0 - if (target.length === 0 || this.length === 0) return 0 - - // Fatal error conditions - if (targetStart < 0) { - throw new RangeError('targetStart out of bounds') - } - if (start < 0 || start >= this.length) throw new RangeError('sourceStart out of bounds') - if (end < 0) throw new RangeError('sourceEnd out of bounds') - - // Are we oob? - if (end > this.length) end = this.length; - if (target.length - targetStart < end - start) { - end = target.length - targetStart + start; - } - - var len = end - start; - var i; - - if (this === target && start < targetStart && targetStart < end) { - // descending copy from end - for (i = len - 1; i >= 0; --i) { - target[i + targetStart] = this[i + start]; - } - } else if (len < 1000 || !Buffer.TYPED_ARRAY_SUPPORT) { - // ascending copy from start - for (i = 0; i < len; ++i) { - target[i + targetStart] = this[i + start]; - } - } else { - Uint8Array.prototype.set.call( - target, - this.subarray(start, start + len), - targetStart - ); - } - - return len - }; - - // Usage: - // buffer.fill(number[, offset[, end]]) - // buffer.fill(buffer[, offset[, end]]) - // buffer.fill(string[, offset[, end]][, encoding]) - Buffer.prototype.fill = function fill (val, start, end, encoding) { - // Handle string cases: - if (typeof val === 'string') { - if (typeof start === 'string') { - encoding = start; - start = 0; - end = this.length; - } else if (typeof end === 'string') { - encoding = end; - end = this.length; - } - if (val.length === 1) { - var code = val.charCodeAt(0); - if (code < 256) { - val = code; - } - } - if (encoding !== undefined && typeof encoding !== 'string') { - throw new TypeError('encoding must be a string') - } - if (typeof encoding === 'string' && !Buffer.isEncoding(encoding)) { - throw new TypeError('Unknown encoding: ' + encoding) - } - } else if (typeof val === 'number') { - val = val & 255; - } - - // Invalid ranges are not set to a default, so can range check early. - if (start < 0 || this.length < start || this.length < end) { - throw new RangeError('Out of range index') - } - - if (end <= start) { - return this - } - - start = start >>> 0; - end = end === undefined ? this.length : end >>> 0; - - if (!val) val = 0; - - var i; - if (typeof val === 'number') { - for (i = start; i < end; ++i) { - this[i] = val; - } - } else { - var bytes = internalIsBuffer(val) - ? val - : utf8ToBytes(new Buffer(val, encoding).toString()); - var len = bytes.length; - for (i = 0; i < end - start; ++i) { - this[i + start] = bytes[i % len]; - } - } - - return this - }; - - // HELPER FUNCTIONS - // ================ - - var INVALID_BASE64_RE = /[^+\/0-9A-Za-z-_]/g; - - function base64clean (str) { - // Node strips out invalid characters like \n and \t from the string, base64-js does not - str = stringtrim(str).replace(INVALID_BASE64_RE, ''); - // Node converts strings with length < 2 to '' - if (str.length < 2) return '' - // Node allows for non-padded base64 strings (missing trailing ===), base64-js does not - while (str.length % 4 !== 0) { - str = str + '='; - } - return str - } - - function stringtrim (str) { - if (str.trim) return str.trim() - return str.replace(/^\s+|\s+$/g, '') - } - - function toHex (n) { - if (n < 16) return '0' + n.toString(16) - return n.toString(16) - } - - function utf8ToBytes (string, units) { - units = units || Infinity; - var codePoint; - var length = string.length; - var leadSurrogate = null; - var bytes = []; - - for (var i = 0; i < length; ++i) { - codePoint = string.charCodeAt(i); - - // is surrogate component - if (codePoint > 0xD7FF && codePoint < 0xE000) { - // last char was a lead - if (!leadSurrogate) { - // no lead yet - if (codePoint > 0xDBFF) { - // unexpected trail - if ((units -= 3) > -1) bytes.push(0xEF, 0xBF, 0xBD); - continue - } else if (i + 1 === length) { - // unpaired lead - if ((units -= 3) > -1) bytes.push(0xEF, 0xBF, 0xBD); - continue - } - - // valid lead - leadSurrogate = codePoint; - - continue - } - - // 2 leads in a row - if (codePoint < 0xDC00) { - if ((units -= 3) > -1) bytes.push(0xEF, 0xBF, 0xBD); - leadSurrogate = codePoint; - continue - } - - // valid surrogate pair - codePoint = (leadSurrogate - 0xD800 << 10 | codePoint - 0xDC00) + 0x10000; - } else if (leadSurrogate) { - // valid bmp char, but last char was a lead - if ((units -= 3) > -1) bytes.push(0xEF, 0xBF, 0xBD); - } - - leadSurrogate = null; - - // encode utf8 - if (codePoint < 0x80) { - if ((units -= 1) < 0) break - bytes.push(codePoint); - } else if (codePoint < 0x800) { - if ((units -= 2) < 0) break - bytes.push( - codePoint >> 0x6 | 0xC0, - codePoint & 0x3F | 0x80 - ); - } else if (codePoint < 0x10000) { - if ((units -= 3) < 0) break - bytes.push( - codePoint >> 0xC | 0xE0, - codePoint >> 0x6 & 0x3F | 0x80, - codePoint & 0x3F | 0x80 - ); - } else if (codePoint < 0x110000) { - if ((units -= 4) < 0) break - bytes.push( - codePoint >> 0x12 | 0xF0, - codePoint >> 0xC & 0x3F | 0x80, - codePoint >> 0x6 & 0x3F | 0x80, - codePoint & 0x3F | 0x80 - ); - } else { - throw new Error('Invalid code point') - } - } - - return bytes - } - - function asciiToBytes (str) { - var byteArray = []; - for (var i = 0; i < str.length; ++i) { - // Node's code seems to be doing this and not & 0x7F.. - byteArray.push(str.charCodeAt(i) & 0xFF); - } - return byteArray - } - - function utf16leToBytes (str, units) { - var c, hi, lo; - var byteArray = []; - for (var i = 0; i < str.length; ++i) { - if ((units -= 2) < 0) break - - c = str.charCodeAt(i); - hi = c >> 8; - lo = c % 256; - byteArray.push(lo); - byteArray.push(hi); - } - - return byteArray - } - - - function base64ToBytes (str) { - return toByteArray(base64clean(str)) - } - - function blitBuffer (src, dst, offset, length) { - for (var i = 0; i < length; ++i) { - if ((i + offset >= dst.length) || (i >= src.length)) break - dst[i + offset] = src[i]; - } - return i - } - - function isnan (val) { - return val !== val // eslint-disable-line no-self-compare - } - - - // the following is from is-buffer, also by Feross Aboukhadijeh and with same lisence - // The _isBuffer check is for Safari 5-7 support, because it's missing - // Object.prototype.constructor. Remove this eventually - function isBuffer(obj) { - return obj != null && (!!obj._isBuffer || isFastBuffer(obj) || isSlowBuffer(obj)) - } - - function isFastBuffer (obj) { - return !!obj.constructor && typeof obj.constructor.isBuffer === 'function' && obj.constructor.isBuffer(obj) - } - - // For Node v0.10 support. Remove this eventually. - function isSlowBuffer (obj) { - return typeof obj.readFloatLE === 'function' && typeof obj.slice === 'function' && isFastBuffer(obj.slice(0, 0)) - } - - var domain; - - // This constructor is used to store event handlers. Instantiating this is - // faster than explicitly calling `Object.create(null)` to get a "clean" empty - // object (tested with v8 v4.9). - function EventHandlers() {} - EventHandlers.prototype = Object.create(null); - - function EventEmitter() { - EventEmitter.init.call(this); - } - - // nodejs oddity - // require('events') === require('events').EventEmitter - EventEmitter.EventEmitter = EventEmitter; - - EventEmitter.usingDomains = false; - - EventEmitter.prototype.domain = undefined; - EventEmitter.prototype._events = undefined; - EventEmitter.prototype._maxListeners = undefined; - - // By default EventEmitters will print a warning if more than 10 listeners are - // added to it. This is a useful default which helps finding memory leaks. - EventEmitter.defaultMaxListeners = 10; - - EventEmitter.init = function() { - this.domain = null; - if (EventEmitter.usingDomains) { - // if there is an active domain, then attach to it. - if (domain.active ) ; - } - - if (!this._events || this._events === Object.getPrototypeOf(this)._events) { - this._events = new EventHandlers(); - this._eventsCount = 0; - } - - this._maxListeners = this._maxListeners || undefined; - }; - - // Obviously not all Emitters should be limited to 10. This function allows - // that to be increased. Set to zero for unlimited. - EventEmitter.prototype.setMaxListeners = function setMaxListeners(n) { - if (typeof n !== 'number' || n < 0 || isNaN(n)) - throw new TypeError('"n" argument must be a positive number'); - this._maxListeners = n; - return this; - }; - - function $getMaxListeners(that) { - if (that._maxListeners === undefined) - return EventEmitter.defaultMaxListeners; - return that._maxListeners; - } - - EventEmitter.prototype.getMaxListeners = function getMaxListeners() { - return $getMaxListeners(this); - }; - - // These standalone emit* functions are used to optimize calling of event - // handlers for fast cases because emit() itself often has a variable number of - // arguments and can be deoptimized because of that. These functions always have - // the same number of arguments and thus do not get deoptimized, so the code - // inside them can execute faster. - function emitNone(handler, isFn, self) { - if (isFn) - handler.call(self); - else { - var len = handler.length; - var listeners = arrayClone(handler, len); - for (var i = 0; i < len; ++i) - listeners[i].call(self); - } - } - function emitOne(handler, isFn, self, arg1) { - if (isFn) - handler.call(self, arg1); - else { - var len = handler.length; - var listeners = arrayClone(handler, len); - for (var i = 0; i < len; ++i) - listeners[i].call(self, arg1); - } - } - function emitTwo(handler, isFn, self, arg1, arg2) { - if (isFn) - handler.call(self, arg1, arg2); - else { - var len = handler.length; - var listeners = arrayClone(handler, len); - for (var i = 0; i < len; ++i) - listeners[i].call(self, arg1, arg2); - } - } - function emitThree(handler, isFn, self, arg1, arg2, arg3) { - if (isFn) - handler.call(self, arg1, arg2, arg3); - else { - var len = handler.length; - var listeners = arrayClone(handler, len); - for (var i = 0; i < len; ++i) - listeners[i].call(self, arg1, arg2, arg3); - } - } - - function emitMany(handler, isFn, self, args) { - if (isFn) - handler.apply(self, args); - else { - var len = handler.length; - var listeners = arrayClone(handler, len); - for (var i = 0; i < len; ++i) - listeners[i].apply(self, args); - } - } - - EventEmitter.prototype.emit = function emit(type) { - var er, handler, len, args, i, events, domain; - var doError = (type === 'error'); - - events = this._events; - if (events) - doError = (doError && events.error == null); - else if (!doError) - return false; - - domain = this.domain; - - // If there is no 'error' event listener then throw. - if (doError) { - er = arguments[1]; - if (domain) { - if (!er) - er = new Error('Uncaught, unspecified "error" event'); - er.domainEmitter = this; - er.domain = domain; - er.domainThrown = false; - domain.emit('error', er); - } else if (er instanceof Error) { - throw er; // Unhandled 'error' event - } else { - // At least give some kind of context to the user - var err = new Error('Uncaught, unspecified "error" event. (' + er + ')'); - err.context = er; - throw err; - } - return false; - } - - handler = events[type]; - - if (!handler) - return false; - - var isFn = typeof handler === 'function'; - len = arguments.length; - switch (len) { - // fast cases - case 1: - emitNone(handler, isFn, this); - break; - case 2: - emitOne(handler, isFn, this, arguments[1]); - break; - case 3: - emitTwo(handler, isFn, this, arguments[1], arguments[2]); - break; - case 4: - emitThree(handler, isFn, this, arguments[1], arguments[2], arguments[3]); - break; - // slower - default: - args = new Array(len - 1); - for (i = 1; i < len; i++) - args[i - 1] = arguments[i]; - emitMany(handler, isFn, this, args); - } - - return true; - }; - - function _addListener(target, type, listener, prepend) { - var m; - var events; - var existing; - - if (typeof listener !== 'function') - throw new TypeError('"listener" argument must be a function'); - - events = target._events; - if (!events) { - events = target._events = new EventHandlers(); - target._eventsCount = 0; - } else { - // To avoid recursion in the case that type === "newListener"! Before - // adding it to the listeners, first emit "newListener". - if (events.newListener) { - target.emit('newListener', type, - listener.listener ? listener.listener : listener); - - // Re-assign `events` because a newListener handler could have caused the - // this._events to be assigned to a new object - events = target._events; - } - existing = events[type]; - } - - if (!existing) { - // Optimize the case of one listener. Don't need the extra array object. - existing = events[type] = listener; - ++target._eventsCount; - } else { - if (typeof existing === 'function') { - // Adding the second element, need to change to array. - existing = events[type] = prepend ? [listener, existing] : - [existing, listener]; - } else { - // If we've already got an array, just append. - if (prepend) { - existing.unshift(listener); - } else { - existing.push(listener); - } - } - - // Check for listener leak - if (!existing.warned) { - m = $getMaxListeners(target); - if (m && m > 0 && existing.length > m) { - existing.warned = true; - var w = new Error('Possible EventEmitter memory leak detected. ' + - existing.length + ' ' + type + ' listeners added. ' + - 'Use emitter.setMaxListeners() to increase limit'); - w.name = 'MaxListenersExceededWarning'; - w.emitter = target; - w.type = type; - w.count = existing.length; - emitWarning(w); - } - } - } - - return target; - } - function emitWarning(e) { - typeof console.warn === 'function' ? console.warn(e) : console.log(e); - } - EventEmitter.prototype.addListener = function addListener(type, listener) { - return _addListener(this, type, listener, false); - }; - - EventEmitter.prototype.on = EventEmitter.prototype.addListener; - - EventEmitter.prototype.prependListener = - function prependListener(type, listener) { - return _addListener(this, type, listener, true); - }; - - function _onceWrap(target, type, listener) { - var fired = false; - function g() { - target.removeListener(type, g); - if (!fired) { - fired = true; - listener.apply(target, arguments); - } - } - g.listener = listener; - return g; - } - - EventEmitter.prototype.once = function once(type, listener) { - if (typeof listener !== 'function') - throw new TypeError('"listener" argument must be a function'); - this.on(type, _onceWrap(this, type, listener)); - return this; - }; - - EventEmitter.prototype.prependOnceListener = - function prependOnceListener(type, listener) { - if (typeof listener !== 'function') - throw new TypeError('"listener" argument must be a function'); - this.prependListener(type, _onceWrap(this, type, listener)); - return this; - }; - - // emits a 'removeListener' event iff the listener was removed - EventEmitter.prototype.removeListener = - function removeListener(type, listener) { - var list, events, position, i, originalListener; - - if (typeof listener !== 'function') - throw new TypeError('"listener" argument must be a function'); - - events = this._events; - if (!events) - return this; - - list = events[type]; - if (!list) - return this; - - if (list === listener || (list.listener && list.listener === listener)) { - if (--this._eventsCount === 0) - this._events = new EventHandlers(); - else { - delete events[type]; - if (events.removeListener) - this.emit('removeListener', type, list.listener || listener); - } - } else if (typeof list !== 'function') { - position = -1; - - for (i = list.length; i-- > 0;) { - if (list[i] === listener || - (list[i].listener && list[i].listener === listener)) { - originalListener = list[i].listener; - position = i; - break; - } - } - - if (position < 0) - return this; - - if (list.length === 1) { - list[0] = undefined; - if (--this._eventsCount === 0) { - this._events = new EventHandlers(); - return this; - } else { - delete events[type]; - } - } else { - spliceOne(list, position); - } - - if (events.removeListener) - this.emit('removeListener', type, originalListener || listener); - } - - return this; - }; - - EventEmitter.prototype.removeAllListeners = - function removeAllListeners(type) { - var listeners, events; - - events = this._events; - if (!events) - return this; - - // not listening for removeListener, no need to emit - if (!events.removeListener) { - if (arguments.length === 0) { - this._events = new EventHandlers(); - this._eventsCount = 0; - } else if (events[type]) { - if (--this._eventsCount === 0) - this._events = new EventHandlers(); - else - delete events[type]; - } - return this; - } - - // emit removeListener for all listeners on all events - if (arguments.length === 0) { - var keys = Object.keys(events); - for (var i = 0, key; i < keys.length; ++i) { - key = keys[i]; - if (key === 'removeListener') continue; - this.removeAllListeners(key); - } - this.removeAllListeners('removeListener'); - this._events = new EventHandlers(); - this._eventsCount = 0; - return this; - } - - listeners = events[type]; - - if (typeof listeners === 'function') { - this.removeListener(type, listeners); - } else if (listeners) { - // LIFO order - do { - this.removeListener(type, listeners[listeners.length - 1]); - } while (listeners[0]); - } - - return this; - }; - - EventEmitter.prototype.listeners = function listeners(type) { - var evlistener; - var ret; - var events = this._events; - - if (!events) - ret = []; - else { - evlistener = events[type]; - if (!evlistener) - ret = []; - else if (typeof evlistener === 'function') - ret = [evlistener.listener || evlistener]; - else - ret = unwrapListeners(evlistener); - } - - return ret; - }; - - EventEmitter.listenerCount = function(emitter, type) { - if (typeof emitter.listenerCount === 'function') { - return emitter.listenerCount(type); - } else { - return listenerCount$1.call(emitter, type); - } - }; - - EventEmitter.prototype.listenerCount = listenerCount$1; - function listenerCount$1(type) { - var events = this._events; - - if (events) { - var evlistener = events[type]; - - if (typeof evlistener === 'function') { - return 1; - } else if (evlistener) { - return evlistener.length; - } - } - - return 0; - } - - EventEmitter.prototype.eventNames = function eventNames() { - return this._eventsCount > 0 ? Reflect.ownKeys(this._events) : []; - }; - - // About 1.5x faster than the two-arg version of Array#splice(). - function spliceOne(list, index) { - for (var i = index, k = i + 1, n = list.length; k < n; i += 1, k += 1) - list[i] = list[k]; - list.pop(); - } - - function arrayClone(arr, i) { - var copy = new Array(i); - while (i--) - copy[i] = arr[i]; - return copy; - } - - function unwrapListeners(arr) { - var ret = new Array(arr.length); - for (var i = 0; i < ret.length; ++i) { - ret[i] = arr[i].listener || arr[i]; - } - return ret; - } - - // shim for using process in browser - // based off https://github.com/defunctzombie/node-process/blob/master/browser.js - - function defaultSetTimout() { - throw new Error('setTimeout has not been defined'); - } - function defaultClearTimeout () { - throw new Error('clearTimeout has not been defined'); - } - var cachedSetTimeout = defaultSetTimout; - var cachedClearTimeout = defaultClearTimeout; - if (typeof global$1.setTimeout === 'function') { - cachedSetTimeout = setTimeout; - } - if (typeof global$1.clearTimeout === 'function') { - cachedClearTimeout = clearTimeout; - } - - function runTimeout(fun) { - if (cachedSetTimeout === setTimeout) { - //normal enviroments in sane situations - return setTimeout(fun, 0); - } - // if setTimeout wasn't available but was latter defined - if ((cachedSetTimeout === defaultSetTimout || !cachedSetTimeout) && setTimeout) { - cachedSetTimeout = setTimeout; - return setTimeout(fun, 0); - } - try { - // when when somebody has screwed with setTimeout but no I.E. maddness - return cachedSetTimeout(fun, 0); - } catch(e){ - try { - // When we are in I.E. but the script has been evaled so I.E. doesn't trust the global object when called normally - return cachedSetTimeout.call(null, fun, 0); - } catch(e){ - // same as above but when it's a version of I.E. that must have the global object for 'this', hopfully our context correct otherwise it will throw a global error - return cachedSetTimeout.call(this, fun, 0); - } - } - - - } - function runClearTimeout(marker) { - if (cachedClearTimeout === clearTimeout) { - //normal enviroments in sane situations - return clearTimeout(marker); - } - // if clearTimeout wasn't available but was latter defined - if ((cachedClearTimeout === defaultClearTimeout || !cachedClearTimeout) && clearTimeout) { - cachedClearTimeout = clearTimeout; - return clearTimeout(marker); - } - try { - // when when somebody has screwed with setTimeout but no I.E. maddness - return cachedClearTimeout(marker); - } catch (e){ - try { - // When we are in I.E. but the script has been evaled so I.E. doesn't trust the global object when called normally - return cachedClearTimeout.call(null, marker); - } catch (e){ - // same as above but when it's a version of I.E. that must have the global object for 'this', hopfully our context correct otherwise it will throw a global error. - // Some versions of I.E. have different rules for clearTimeout vs setTimeout - return cachedClearTimeout.call(this, marker); - } - } - - - - } - var queue = []; - var draining = false; - var currentQueue; - var queueIndex = -1; - - function cleanUpNextTick() { - if (!draining || !currentQueue) { - return; - } - draining = false; - if (currentQueue.length) { - queue = currentQueue.concat(queue); - } else { - queueIndex = -1; - } - if (queue.length) { - drainQueue(); - } - } - - function drainQueue() { - if (draining) { - return; - } - var timeout = runTimeout(cleanUpNextTick); - draining = true; - - var len = queue.length; - while(len) { - currentQueue = queue; - queue = []; - while (++queueIndex < len) { - if (currentQueue) { - currentQueue[queueIndex].run(); - } - } - queueIndex = -1; - len = queue.length; - } - currentQueue = null; - draining = false; - runClearTimeout(timeout); - } - function nextTick(fun) { - var args = new Array(arguments.length - 1); - if (arguments.length > 1) { - for (var i = 1; i < arguments.length; i++) { - args[i - 1] = arguments[i]; - } - } - queue.push(new Item(fun, args)); - if (queue.length === 1 && !draining) { - runTimeout(drainQueue); - } - } - // v8 likes predictible objects - function Item(fun, array) { - this.fun = fun; - this.array = array; - } - Item.prototype.run = function () { - this.fun.apply(null, this.array); - }; - - // from https://github.com/kumavis/browser-process-hrtime/blob/master/index.js - var performance = global$1.performance || {}; - performance.now || - performance.mozNow || - performance.msNow || - performance.oNow || - performance.webkitNow || - function(){ return (new Date()).getTime() }; - - var inherits; - if (typeof Object.create === 'function'){ - inherits = function inherits(ctor, superCtor) { - // implementation from standard node.js 'util' module - ctor.super_ = superCtor; - ctor.prototype = Object.create(superCtor.prototype, { - constructor: { - value: ctor, - enumerable: false, - writable: true, - configurable: true - } - }); - }; - } else { - inherits = function inherits(ctor, superCtor) { - ctor.super_ = superCtor; - var TempCtor = function () {}; - TempCtor.prototype = superCtor.prototype; - ctor.prototype = new TempCtor(); - ctor.prototype.constructor = ctor; - }; - } - var inherits$1 = inherits; - - var formatRegExp = /%[sdj%]/g; - function format(f) { - if (!isString(f)) { - var objects = []; - for (var i = 0; i < arguments.length; i++) { - objects.push(inspect(arguments[i])); - } - return objects.join(' '); - } - - var i = 1; - var args = arguments; - var len = args.length; - var str = String(f).replace(formatRegExp, function(x) { - if (x === '%%') return '%'; - if (i >= len) return x; - switch (x) { - case '%s': return String(args[i++]); - case '%d': return Number(args[i++]); - case '%j': - try { - return JSON.stringify(args[i++]); - } catch (_) { - return '[Circular]'; - } - default: - return x; - } - }); - for (var x = args[i]; i < len; x = args[++i]) { - if (isNull(x) || !isObject$1(x)) { - str += ' ' + x; - } else { - str += ' ' + inspect(x); - } - } - return str; - } - - // Mark that a method should not be used. - // Returns a modified function which warns once by default. - // If --no-deprecation is set, then it is a no-op. - function deprecate(fn, msg) { - // Allow for deprecating things in the process of starting up. - if (isUndefined(global$1.process)) { - return function() { - return deprecate(fn, msg).apply(this, arguments); - }; - } - - var warned = false; - function deprecated() { - if (!warned) { - { - console.error(msg); - } - warned = true; - } - return fn.apply(this, arguments); - } - - return deprecated; - } - - var debugs = {}; - var debugEnviron; - function debuglog(set) { - if (isUndefined(debugEnviron)) - debugEnviron = ''; - set = set.toUpperCase(); - if (!debugs[set]) { - if (new RegExp('\\b' + set + '\\b', 'i').test(debugEnviron)) { - var pid = 0; - debugs[set] = function() { - var msg = format.apply(null, arguments); - console.error('%s %d: %s', set, pid, msg); - }; - } else { - debugs[set] = function() {}; - } - } - return debugs[set]; - } - - /** - * Echos the value of a value. Trys to print the value out - * in the best way possible given the different types. - * - * @param {Object} obj The object to print out. - * @param {Object} opts Optional options object that alters the output. - */ - /* legacy: obj, showHidden, depth, colors*/ - function inspect(obj, opts) { - // default options - var ctx = { - seen: [], - stylize: stylizeNoColor - }; - // legacy... - if (arguments.length >= 3) ctx.depth = arguments[2]; - if (arguments.length >= 4) ctx.colors = arguments[3]; - if (isBoolean(opts)) { - // legacy... - ctx.showHidden = opts; - } else if (opts) { - // got an "options" object - _extend(ctx, opts); - } - // set default options - if (isUndefined(ctx.showHidden)) ctx.showHidden = false; - if (isUndefined(ctx.depth)) ctx.depth = 2; - if (isUndefined(ctx.colors)) ctx.colors = false; - if (isUndefined(ctx.customInspect)) ctx.customInspect = true; - if (ctx.colors) ctx.stylize = stylizeWithColor; - return formatValue(ctx, obj, ctx.depth); - } - - // http://en.wikipedia.org/wiki/ANSI_escape_code#graphics - inspect.colors = { - 'bold' : [1, 22], - 'italic' : [3, 23], - 'underline' : [4, 24], - 'inverse' : [7, 27], - 'white' : [37, 39], - 'grey' : [90, 39], - 'black' : [30, 39], - 'blue' : [34, 39], - 'cyan' : [36, 39], - 'green' : [32, 39], - 'magenta' : [35, 39], - 'red' : [31, 39], - 'yellow' : [33, 39] - }; - - // Don't use 'blue' not visible on cmd.exe - inspect.styles = { - 'special': 'cyan', - 'number': 'yellow', - 'boolean': 'yellow', - 'undefined': 'grey', - 'null': 'bold', - 'string': 'green', - 'date': 'magenta', - // "name": intentionally not styling - 'regexp': 'red' - }; - - - function stylizeWithColor(str, styleType) { - var style = inspect.styles[styleType]; - - if (style) { - return '\u001b[' + inspect.colors[style][0] + 'm' + str + - '\u001b[' + inspect.colors[style][1] + 'm'; - } else { - return str; - } - } - - - function stylizeNoColor(str, styleType) { - return str; - } - - - function arrayToHash(array) { - var hash = {}; - - array.forEach(function(val, idx) { - hash[val] = true; - }); - - return hash; - } - - - function formatValue(ctx, value, recurseTimes) { - // Provide a hook for user-specified inspect functions. - // Check that value is an object with an inspect function on it - if (ctx.customInspect && - value && - isFunction(value.inspect) && - // Filter out the util module, it's inspect function is special - value.inspect !== inspect && - // Also filter out any prototype objects using the circular check. - !(value.constructor && value.constructor.prototype === value)) { - var ret = value.inspect(recurseTimes, ctx); - if (!isString(ret)) { - ret = formatValue(ctx, ret, recurseTimes); - } - return ret; - } - - // Primitive types cannot have properties - var primitive = formatPrimitive(ctx, value); - if (primitive) { - return primitive; - } - - // Look up the keys of the object. - var keys = Object.keys(value); - var visibleKeys = arrayToHash(keys); - - if (ctx.showHidden) { - keys = Object.getOwnPropertyNames(value); - } - - // IE doesn't make error fields non-enumerable - // http://msdn.microsoft.com/en-us/library/ie/dww52sbt(v=vs.94).aspx - if (isError(value) - && (keys.indexOf('message') >= 0 || keys.indexOf('description') >= 0)) { - return formatError(value); - } - - // Some type of object without properties can be shortcutted. - if (keys.length === 0) { - if (isFunction(value)) { - var name = value.name ? ': ' + value.name : ''; - return ctx.stylize('[Function' + name + ']', 'special'); - } - if (isRegExp(value)) { - return ctx.stylize(RegExp.prototype.toString.call(value), 'regexp'); - } - if (isDate(value)) { - return ctx.stylize(Date.prototype.toString.call(value), 'date'); - } - if (isError(value)) { - return formatError(value); - } - } - - var base = '', array = false, braces = ['{', '}']; - - // Make Array say that they are Array - if (isArray(value)) { - array = true; - braces = ['[', ']']; - } - - // Make functions say that they are functions - if (isFunction(value)) { - var n = value.name ? ': ' + value.name : ''; - base = ' [Function' + n + ']'; - } - - // Make RegExps say that they are RegExps - if (isRegExp(value)) { - base = ' ' + RegExp.prototype.toString.call(value); - } - - // Make dates with properties first say the date - if (isDate(value)) { - base = ' ' + Date.prototype.toUTCString.call(value); - } - - // Make error with message first say the error - if (isError(value)) { - base = ' ' + formatError(value); - } - - if (keys.length === 0 && (!array || value.length == 0)) { - return braces[0] + base + braces[1]; - } - - if (recurseTimes < 0) { - if (isRegExp(value)) { - return ctx.stylize(RegExp.prototype.toString.call(value), 'regexp'); - } else { - return ctx.stylize('[Object]', 'special'); - } - } - - ctx.seen.push(value); - - var output; - if (array) { - output = formatArray(ctx, value, recurseTimes, visibleKeys, keys); - } else { - output = keys.map(function(key) { - return formatProperty(ctx, value, recurseTimes, visibleKeys, key, array); - }); - } - - ctx.seen.pop(); - - return reduceToSingleString(output, base, braces); - } - - - function formatPrimitive(ctx, value) { - if (isUndefined(value)) - return ctx.stylize('undefined', 'undefined'); - if (isString(value)) { - var simple = '\'' + JSON.stringify(value).replace(/^"|"$/g, '') - .replace(/'/g, "\\'") - .replace(/\\"/g, '"') + '\''; - return ctx.stylize(simple, 'string'); - } - if (isNumber(value)) - return ctx.stylize('' + value, 'number'); - if (isBoolean(value)) - return ctx.stylize('' + value, 'boolean'); - // For some reason typeof null is "object", so special case here. - if (isNull(value)) - return ctx.stylize('null', 'null'); - } - - - function formatError(value) { - return '[' + Error.prototype.toString.call(value) + ']'; - } - - - function formatArray(ctx, value, recurseTimes, visibleKeys, keys) { - var output = []; - for (var i = 0, l = value.length; i < l; ++i) { - if (hasOwnProperty(value, String(i))) { - output.push(formatProperty(ctx, value, recurseTimes, visibleKeys, - String(i), true)); - } else { - output.push(''); - } - } - keys.forEach(function(key) { - if (!key.match(/^\d+$/)) { - output.push(formatProperty(ctx, value, recurseTimes, visibleKeys, - key, true)); - } - }); - return output; - } - - - function formatProperty(ctx, value, recurseTimes, visibleKeys, key, array) { - var name, str, desc; - desc = Object.getOwnPropertyDescriptor(value, key) || { value: value[key] }; - if (desc.get) { - if (desc.set) { - str = ctx.stylize('[Getter/Setter]', 'special'); - } else { - str = ctx.stylize('[Getter]', 'special'); - } - } else { - if (desc.set) { - str = ctx.stylize('[Setter]', 'special'); - } - } - if (!hasOwnProperty(visibleKeys, key)) { - name = '[' + key + ']'; - } - if (!str) { - if (ctx.seen.indexOf(desc.value) < 0) { - if (isNull(recurseTimes)) { - str = formatValue(ctx, desc.value, null); - } else { - str = formatValue(ctx, desc.value, recurseTimes - 1); - } - if (str.indexOf('\n') > -1) { - if (array) { - str = str.split('\n').map(function(line) { - return ' ' + line; - }).join('\n').substr(2); - } else { - str = '\n' + str.split('\n').map(function(line) { - return ' ' + line; - }).join('\n'); - } - } - } else { - str = ctx.stylize('[Circular]', 'special'); - } - } - if (isUndefined(name)) { - if (array && key.match(/^\d+$/)) { - return str; - } - name = JSON.stringify('' + key); - if (name.match(/^"([a-zA-Z_][a-zA-Z_0-9]*)"$/)) { - name = name.substr(1, name.length - 2); - name = ctx.stylize(name, 'name'); - } else { - name = name.replace(/'/g, "\\'") - .replace(/\\"/g, '"') - .replace(/(^"|"$)/g, "'"); - name = ctx.stylize(name, 'string'); - } - } - - return name + ': ' + str; - } - - - function reduceToSingleString(output, base, braces) { - var length = output.reduce(function(prev, cur) { - if (cur.indexOf('\n') >= 0) ; - return prev + cur.replace(/\u001b\[\d\d?m/g, '').length + 1; - }, 0); - - if (length > 60) { - return braces[0] + - (base === '' ? '' : base + '\n ') + - ' ' + - output.join(',\n ') + - ' ' + - braces[1]; - } - - return braces[0] + base + ' ' + output.join(', ') + ' ' + braces[1]; - } - - - // NOTE: These type checking functions intentionally don't use `instanceof` - // because it is fragile and can be easily faked with `Object.create()`. - function isArray(ar) { - return Array.isArray(ar); - } - - function isBoolean(arg) { - return typeof arg === 'boolean'; - } - - function isNull(arg) { - return arg === null; - } - - function isNumber(arg) { - return typeof arg === 'number'; - } - - function isString(arg) { - return typeof arg === 'string'; - } - - function isUndefined(arg) { - return arg === void 0; - } - - function isRegExp(re) { - return isObject$1(re) && objectToString(re) === '[object RegExp]'; - } - - function isObject$1(arg) { - return typeof arg === 'object' && arg !== null; - } - - function isDate(d) { - return isObject$1(d) && objectToString(d) === '[object Date]'; - } - - function isError(e) { - return isObject$1(e) && - (objectToString(e) === '[object Error]' || e instanceof Error); - } - - function isFunction(arg) { - return typeof arg === 'function'; - } - - function objectToString(o) { - return Object.prototype.toString.call(o); - } - - function _extend(origin, add) { - // Don't do anything if add isn't an object - if (!add || !isObject$1(add)) return origin; - - var keys = Object.keys(add); - var i = keys.length; - while (i--) { - origin[keys[i]] = add[keys[i]]; - } - return origin; - } - function hasOwnProperty(obj, prop) { - return Object.prototype.hasOwnProperty.call(obj, prop); - } - - function BufferList() { - this.head = null; - this.tail = null; - this.length = 0; - } - - BufferList.prototype.push = function (v) { - var entry = { data: v, next: null }; - if (this.length > 0) this.tail.next = entry;else this.head = entry; - this.tail = entry; - ++this.length; - }; - - BufferList.prototype.unshift = function (v) { - var entry = { data: v, next: this.head }; - if (this.length === 0) this.tail = entry; - this.head = entry; - ++this.length; - }; - - BufferList.prototype.shift = function () { - if (this.length === 0) return; - var ret = this.head.data; - if (this.length === 1) this.head = this.tail = null;else this.head = this.head.next; - --this.length; - return ret; - }; - - BufferList.prototype.clear = function () { - this.head = this.tail = null; - this.length = 0; - }; - - BufferList.prototype.join = function (s) { - if (this.length === 0) return ''; - var p = this.head; - var ret = '' + p.data; - while (p = p.next) { - ret += s + p.data; - }return ret; - }; - - BufferList.prototype.concat = function (n) { - if (this.length === 0) return Buffer.alloc(0); - if (this.length === 1) return this.head.data; - var ret = Buffer.allocUnsafe(n >>> 0); - var p = this.head; - var i = 0; - while (p) { - p.data.copy(ret, i); - i += p.data.length; - p = p.next; - } - return ret; - }; - - // Copyright Joyent, Inc. and other Node contributors. - var isBufferEncoding = Buffer.isEncoding - || function(encoding) { - switch (encoding && encoding.toLowerCase()) { - case 'hex': case 'utf8': case 'utf-8': case 'ascii': case 'binary': case 'base64': case 'ucs2': case 'ucs-2': case 'utf16le': case 'utf-16le': case 'raw': return true; - default: return false; - } - }; - - - function assertEncoding(encoding) { - if (encoding && !isBufferEncoding(encoding)) { - throw new Error('Unknown encoding: ' + encoding); - } - } - - // StringDecoder provides an interface for efficiently splitting a series of - // buffers into a series of JS strings without breaking apart multi-byte - // characters. CESU-8 is handled as part of the UTF-8 encoding. - // - // @TODO Handling all encodings inside a single object makes it very difficult - // to reason about this code, so it should be split up in the future. - // @TODO There should be a utf8-strict encoding that rejects invalid UTF-8 code - // points as used by CESU-8. - function StringDecoder(encoding) { - this.encoding = (encoding || 'utf8').toLowerCase().replace(/[-_]/, ''); - assertEncoding(encoding); - switch (this.encoding) { - case 'utf8': - // CESU-8 represents each of Surrogate Pair by 3-bytes - this.surrogateSize = 3; - break; - case 'ucs2': - case 'utf16le': - // UTF-16 represents each of Surrogate Pair by 2-bytes - this.surrogateSize = 2; - this.detectIncompleteChar = utf16DetectIncompleteChar; - break; - case 'base64': - // Base-64 stores 3 bytes in 4 chars, and pads the remainder. - this.surrogateSize = 3; - this.detectIncompleteChar = base64DetectIncompleteChar; - break; - default: - this.write = passThroughWrite; - return; - } - - // Enough space to store all bytes of a single character. UTF-8 needs 4 - // bytes, but CESU-8 may require up to 6 (3 bytes per surrogate). - this.charBuffer = new Buffer(6); - // Number of bytes received for the current incomplete multi-byte character. - this.charReceived = 0; - // Number of bytes expected for the current incomplete multi-byte character. - this.charLength = 0; - } - - // write decodes the given buffer and returns it as JS string that is - // guaranteed to not contain any partial multi-byte characters. Any partial - // character found at the end of the buffer is buffered up, and will be - // returned when calling write again with the remaining bytes. - // - // Note: Converting a Buffer containing an orphan surrogate to a String - // currently works, but converting a String to a Buffer (via `new Buffer`, or - // Buffer#write) will replace incomplete surrogates with the unicode - // replacement character. See https://codereview.chromium.org/121173009/ . - StringDecoder.prototype.write = function(buffer) { - var charStr = ''; - // if our last write ended with an incomplete multibyte character - while (this.charLength) { - // determine how many remaining bytes this buffer has to offer for this char - var available = (buffer.length >= this.charLength - this.charReceived) ? - this.charLength - this.charReceived : - buffer.length; - - // add the new bytes to the char buffer - buffer.copy(this.charBuffer, this.charReceived, 0, available); - this.charReceived += available; - - if (this.charReceived < this.charLength) { - // still not enough chars in this buffer? wait for more ... - return ''; - } - - // remove bytes belonging to the current character from the buffer - buffer = buffer.slice(available, buffer.length); - - // get the character that was split - charStr = this.charBuffer.slice(0, this.charLength).toString(this.encoding); - - // CESU-8: lead surrogate (D800-DBFF) is also the incomplete character - var charCode = charStr.charCodeAt(charStr.length - 1); - if (charCode >= 0xD800 && charCode <= 0xDBFF) { - this.charLength += this.surrogateSize; - charStr = ''; - continue; - } - this.charReceived = this.charLength = 0; - - // if there are no more bytes in this buffer, just emit our char - if (buffer.length === 0) { - return charStr; - } - break; - } - - // determine and set charLength / charReceived - this.detectIncompleteChar(buffer); - - var end = buffer.length; - if (this.charLength) { - // buffer the incomplete character bytes we got - buffer.copy(this.charBuffer, 0, buffer.length - this.charReceived, end); - end -= this.charReceived; - } - - charStr += buffer.toString(this.encoding, 0, end); - - var end = charStr.length - 1; - var charCode = charStr.charCodeAt(end); - // CESU-8: lead surrogate (D800-DBFF) is also the incomplete character - if (charCode >= 0xD800 && charCode <= 0xDBFF) { - var size = this.surrogateSize; - this.charLength += size; - this.charReceived += size; - this.charBuffer.copy(this.charBuffer, size, 0, size); - buffer.copy(this.charBuffer, 0, 0, size); - return charStr.substring(0, end); - } - - // or just emit the charStr - return charStr; - }; - - // detectIncompleteChar determines if there is an incomplete UTF-8 character at - // the end of the given buffer. If so, it sets this.charLength to the byte - // length that character, and sets this.charReceived to the number of bytes - // that are available for this character. - StringDecoder.prototype.detectIncompleteChar = function(buffer) { - // determine how many bytes we have to check at the end of this buffer - var i = (buffer.length >= 3) ? 3 : buffer.length; - - // Figure out if one of the last i bytes of our buffer announces an - // incomplete char. - for (; i > 0; i--) { - var c = buffer[buffer.length - i]; - - // See http://en.wikipedia.org/wiki/UTF-8#Description - - // 110XXXXX - if (i == 1 && c >> 5 == 0x06) { - this.charLength = 2; - break; - } - - // 1110XXXX - if (i <= 2 && c >> 4 == 0x0E) { - this.charLength = 3; - break; - } - - // 11110XXX - if (i <= 3 && c >> 3 == 0x1E) { - this.charLength = 4; - break; - } - } - this.charReceived = i; - }; - - StringDecoder.prototype.end = function(buffer) { - var res = ''; - if (buffer && buffer.length) - res = this.write(buffer); - - if (this.charReceived) { - var cr = this.charReceived; - var buf = this.charBuffer; - var enc = this.encoding; - res += buf.slice(0, cr).toString(enc); - } - - return res; - }; - - function passThroughWrite(buffer) { - return buffer.toString(this.encoding); - } - - function utf16DetectIncompleteChar(buffer) { - this.charReceived = buffer.length % 2; - this.charLength = this.charReceived ? 2 : 0; - } - - function base64DetectIncompleteChar(buffer) { - this.charReceived = buffer.length % 3; - this.charLength = this.charReceived ? 3 : 0; - } - - Readable.ReadableState = ReadableState; - - var debug = debuglog('stream'); - inherits$1(Readable, EventEmitter); - - function prependListener(emitter, event, fn) { - // Sadly this is not cacheable as some libraries bundle their own - // event emitter implementation with them. - if (typeof emitter.prependListener === 'function') { - return emitter.prependListener(event, fn); - } else { - // This is a hack to make sure that our error handler is attached before any - // userland ones. NEVER DO THIS. This is here only because this code needs - // to continue to work with older versions of Node.js that do not include - // the prependListener() method. The goal is to eventually remove this hack. - if (!emitter._events || !emitter._events[event]) - emitter.on(event, fn); - else if (Array.isArray(emitter._events[event])) - emitter._events[event].unshift(fn); - else - emitter._events[event] = [fn, emitter._events[event]]; - } - } - function listenerCount (emitter, type) { - return emitter.listeners(type).length; - } - function ReadableState(options, stream) { - - options = options || {}; - - // object stream flag. Used to make read(n) ignore n and to - // make all the buffer merging and length checks go away - this.objectMode = !!options.objectMode; - - if (stream instanceof Duplex) this.objectMode = this.objectMode || !!options.readableObjectMode; - - // the point at which it stops calling _read() to fill the buffer - // Note: 0 is a valid value, means "don't call _read preemptively ever" - var hwm = options.highWaterMark; - var defaultHwm = this.objectMode ? 16 : 16 * 1024; - this.highWaterMark = hwm || hwm === 0 ? hwm : defaultHwm; - - // cast to ints. - this.highWaterMark = ~ ~this.highWaterMark; - - // A linked list is used to store data chunks instead of an array because the - // linked list can remove elements from the beginning faster than - // array.shift() - this.buffer = new BufferList(); - this.length = 0; - this.pipes = null; - this.pipesCount = 0; - this.flowing = null; - this.ended = false; - this.endEmitted = false; - this.reading = false; - - // a flag to be able to tell if the onwrite cb is called immediately, - // or on a later tick. We set this to true at first, because any - // actions that shouldn't happen until "later" should generally also - // not happen before the first write call. - this.sync = true; - - // whenever we return null, then we set a flag to say - // that we're awaiting a 'readable' event emission. - this.needReadable = false; - this.emittedReadable = false; - this.readableListening = false; - this.resumeScheduled = false; - - // Crypto is kind of old and crusty. Historically, its default string - // encoding is 'binary' so we have to make this configurable. - // Everything else in the universe uses 'utf8', though. - this.defaultEncoding = options.defaultEncoding || 'utf8'; - - // when piping, we only care about 'readable' events that happen - // after read()ing all the bytes and not getting any pushback. - this.ranOut = false; - - // the number of writers that are awaiting a drain event in .pipe()s - this.awaitDrain = 0; - - // if true, a maybeReadMore has been scheduled - this.readingMore = false; - - this.decoder = null; - this.encoding = null; - if (options.encoding) { - this.decoder = new StringDecoder(options.encoding); - this.encoding = options.encoding; - } - } - function Readable(options) { - - if (!(this instanceof Readable)) return new Readable(options); - - this._readableState = new ReadableState(options, this); - - // legacy - this.readable = true; - - if (options && typeof options.read === 'function') this._read = options.read; - - EventEmitter.call(this); - } - - // Manually shove something into the read() buffer. - // This returns true if the highWaterMark has not been hit yet, - // similar to how Writable.write() returns true if you should - // write() some more. - Readable.prototype.push = function (chunk, encoding) { - var state = this._readableState; - - if (!state.objectMode && typeof chunk === 'string') { - encoding = encoding || state.defaultEncoding; - if (encoding !== state.encoding) { - chunk = Buffer.from(chunk, encoding); - encoding = ''; - } - } - - return readableAddChunk(this, state, chunk, encoding, false); - }; - - // Unshift should *always* be something directly out of read() - Readable.prototype.unshift = function (chunk) { - var state = this._readableState; - return readableAddChunk(this, state, chunk, '', true); - }; - - Readable.prototype.isPaused = function () { - return this._readableState.flowing === false; - }; - - function readableAddChunk(stream, state, chunk, encoding, addToFront) { - var er = chunkInvalid(state, chunk); - if (er) { - stream.emit('error', er); - } else if (chunk === null) { - state.reading = false; - onEofChunk(stream, state); - } else if (state.objectMode || chunk && chunk.length > 0) { - if (state.ended && !addToFront) { - var e = new Error('stream.push() after EOF'); - stream.emit('error', e); - } else if (state.endEmitted && addToFront) { - var _e = new Error('stream.unshift() after end event'); - stream.emit('error', _e); - } else { - var skipAdd; - if (state.decoder && !addToFront && !encoding) { - chunk = state.decoder.write(chunk); - skipAdd = !state.objectMode && chunk.length === 0; - } - - if (!addToFront) state.reading = false; - - // Don't add to the buffer if we've decoded to an empty string chunk and - // we're not in object mode - if (!skipAdd) { - // if we want the data now, just emit it. - if (state.flowing && state.length === 0 && !state.sync) { - stream.emit('data', chunk); - stream.read(0); - } else { - // update the buffer info. - state.length += state.objectMode ? 1 : chunk.length; - if (addToFront) state.buffer.unshift(chunk);else state.buffer.push(chunk); - - if (state.needReadable) emitReadable(stream); - } - } - - maybeReadMore(stream, state); - } - } else if (!addToFront) { - state.reading = false; - } - - return needMoreData(state); - } - - // if it's past the high water mark, we can push in some more. - // Also, if we have no data yet, we can stand some - // more bytes. This is to work around cases where hwm=0, - // such as the repl. Also, if the push() triggered a - // readable event, and the user called read(largeNumber) such that - // needReadable was set, then we ought to push more, so that another - // 'readable' event will be triggered. - function needMoreData(state) { - return !state.ended && (state.needReadable || state.length < state.highWaterMark || state.length === 0); - } - - // backwards compatibility. - Readable.prototype.setEncoding = function (enc) { - this._readableState.decoder = new StringDecoder(enc); - this._readableState.encoding = enc; - return this; - }; - - // Don't raise the hwm > 8MB - var MAX_HWM = 0x800000; - function computeNewHighWaterMark(n) { - if (n >= MAX_HWM) { - n = MAX_HWM; - } else { - // Get the next highest power of 2 to prevent increasing hwm excessively in - // tiny amounts - n--; - n |= n >>> 1; - n |= n >>> 2; - n |= n >>> 4; - n |= n >>> 8; - n |= n >>> 16; - n++; - } - return n; - } - - // This function is designed to be inlinable, so please take care when making - // changes to the function body. - function howMuchToRead(n, state) { - if (n <= 0 || state.length === 0 && state.ended) return 0; - if (state.objectMode) return 1; - if (n !== n) { - // Only flow one buffer at a time - if (state.flowing && state.length) return state.buffer.head.data.length;else return state.length; - } - // If we're asking for more than the current hwm, then raise the hwm. - if (n > state.highWaterMark) state.highWaterMark = computeNewHighWaterMark(n); - if (n <= state.length) return n; - // Don't have enough - if (!state.ended) { - state.needReadable = true; - return 0; - } - return state.length; - } - - // you can override either this method, or the async _read(n) below. - Readable.prototype.read = function (n) { - debug('read', n); - n = parseInt(n, 10); - var state = this._readableState; - var nOrig = n; - - if (n !== 0) state.emittedReadable = false; - - // if we're doing read(0) to trigger a readable event, but we - // already have a bunch of data in the buffer, then just trigger - // the 'readable' event and move on. - if (n === 0 && state.needReadable && (state.length >= state.highWaterMark || state.ended)) { - debug('read: emitReadable', state.length, state.ended); - if (state.length === 0 && state.ended) endReadable(this);else emitReadable(this); - return null; - } - - n = howMuchToRead(n, state); - - // if we've ended, and we're now clear, then finish it up. - if (n === 0 && state.ended) { - if (state.length === 0) endReadable(this); - return null; - } - - // All the actual chunk generation logic needs to be - // *below* the call to _read. The reason is that in certain - // synthetic stream cases, such as passthrough streams, _read - // may be a completely synchronous operation which may change - // the state of the read buffer, providing enough data when - // before there was *not* enough. - // - // So, the steps are: - // 1. Figure out what the state of things will be after we do - // a read from the buffer. - // - // 2. If that resulting state will trigger a _read, then call _read. - // Note that this may be asynchronous, or synchronous. Yes, it is - // deeply ugly to write APIs this way, but that still doesn't mean - // that the Readable class should behave improperly, as streams are - // designed to be sync/async agnostic. - // Take note if the _read call is sync or async (ie, if the read call - // has returned yet), so that we know whether or not it's safe to emit - // 'readable' etc. - // - // 3. Actually pull the requested chunks out of the buffer and return. - - // if we need a readable event, then we need to do some reading. - var doRead = state.needReadable; - debug('need readable', doRead); - - // if we currently have less than the highWaterMark, then also read some - if (state.length === 0 || state.length - n < state.highWaterMark) { - doRead = true; - debug('length less than watermark', doRead); - } - - // however, if we've ended, then there's no point, and if we're already - // reading, then it's unnecessary. - if (state.ended || state.reading) { - doRead = false; - debug('reading or ended', doRead); - } else if (doRead) { - debug('do read'); - state.reading = true; - state.sync = true; - // if the length is currently zero, then we *need* a readable event. - if (state.length === 0) state.needReadable = true; - // call internal read method - this._read(state.highWaterMark); - state.sync = false; - // If _read pushed data synchronously, then `reading` will be false, - // and we need to re-evaluate how much data we can return to the user. - if (!state.reading) n = howMuchToRead(nOrig, state); - } - - var ret; - if (n > 0) ret = fromList(n, state);else ret = null; - - if (ret === null) { - state.needReadable = true; - n = 0; - } else { - state.length -= n; - } - - if (state.length === 0) { - // If we have nothing in the buffer, then we want to know - // as soon as we *do* get something into the buffer. - if (!state.ended) state.needReadable = true; - - // If we tried to read() past the EOF, then emit end on the next tick. - if (nOrig !== n && state.ended) endReadable(this); - } - - if (ret !== null) this.emit('data', ret); - - return ret; - }; - - function chunkInvalid(state, chunk) { - var er = null; - if (!isBuffer(chunk) && typeof chunk !== 'string' && chunk !== null && chunk !== undefined && !state.objectMode) { - er = new TypeError('Invalid non-string/buffer chunk'); - } - return er; - } - - function onEofChunk(stream, state) { - if (state.ended) return; - if (state.decoder) { - var chunk = state.decoder.end(); - if (chunk && chunk.length) { - state.buffer.push(chunk); - state.length += state.objectMode ? 1 : chunk.length; - } - } - state.ended = true; - - // emit 'readable' now to make sure it gets picked up. - emitReadable(stream); - } - - // Don't emit readable right away in sync mode, because this can trigger - // another read() call => stack overflow. This way, it might trigger - // a nextTick recursion warning, but that's not so bad. - function emitReadable(stream) { - var state = stream._readableState; - state.needReadable = false; - if (!state.emittedReadable) { - debug('emitReadable', state.flowing); - state.emittedReadable = true; - if (state.sync) nextTick(emitReadable_, stream);else emitReadable_(stream); - } - } - - function emitReadable_(stream) { - debug('emit readable'); - stream.emit('readable'); - flow(stream); - } - - // at this point, the user has presumably seen the 'readable' event, - // and called read() to consume some data. that may have triggered - // in turn another _read(n) call, in which case reading = true if - // it's in progress. - // However, if we're not ended, or reading, and the length < hwm, - // then go ahead and try to read some more preemptively. - function maybeReadMore(stream, state) { - if (!state.readingMore) { - state.readingMore = true; - nextTick(maybeReadMore_, stream, state); - } - } - - function maybeReadMore_(stream, state) { - var len = state.length; - while (!state.reading && !state.flowing && !state.ended && state.length < state.highWaterMark) { - debug('maybeReadMore read 0'); - stream.read(0); - if (len === state.length) - // didn't get any data, stop spinning. - break;else len = state.length; - } - state.readingMore = false; - } - - // abstract method. to be overridden in specific implementation classes. - // call cb(er, data) where data is <= n in length. - // for virtual (non-string, non-buffer) streams, "length" is somewhat - // arbitrary, and perhaps not very meaningful. - Readable.prototype._read = function (n) { - this.emit('error', new Error('not implemented')); - }; - - Readable.prototype.pipe = function (dest, pipeOpts) { - var src = this; - var state = this._readableState; - - switch (state.pipesCount) { - case 0: - state.pipes = dest; - break; - case 1: - state.pipes = [state.pipes, dest]; - break; - default: - state.pipes.push(dest); - break; - } - state.pipesCount += 1; - debug('pipe count=%d opts=%j', state.pipesCount, pipeOpts); - - var doEnd = (!pipeOpts || pipeOpts.end !== false); - - var endFn = doEnd ? onend : cleanup; - if (state.endEmitted) nextTick(endFn);else src.once('end', endFn); - - dest.on('unpipe', onunpipe); - function onunpipe(readable) { - debug('onunpipe'); - if (readable === src) { - cleanup(); - } - } - - function onend() { - debug('onend'); - dest.end(); - } - - // when the dest drains, it reduces the awaitDrain counter - // on the source. This would be more elegant with a .once() - // handler in flow(), but adding and removing repeatedly is - // too slow. - var ondrain = pipeOnDrain(src); - dest.on('drain', ondrain); - - var cleanedUp = false; - function cleanup() { - debug('cleanup'); - // cleanup event handlers once the pipe is broken - dest.removeListener('close', onclose); - dest.removeListener('finish', onfinish); - dest.removeListener('drain', ondrain); - dest.removeListener('error', onerror); - dest.removeListener('unpipe', onunpipe); - src.removeListener('end', onend); - src.removeListener('end', cleanup); - src.removeListener('data', ondata); - - cleanedUp = true; - - // if the reader is waiting for a drain event from this - // specific writer, then it would cause it to never start - // flowing again. - // So, if this is awaiting a drain, then we just call it now. - // If we don't know, then assume that we are waiting for one. - if (state.awaitDrain && (!dest._writableState || dest._writableState.needDrain)) ondrain(); - } - - // If the user pushes more data while we're writing to dest then we'll end up - // in ondata again. However, we only want to increase awaitDrain once because - // dest will only emit one 'drain' event for the multiple writes. - // => Introduce a guard on increasing awaitDrain. - var increasedAwaitDrain = false; - src.on('data', ondata); - function ondata(chunk) { - debug('ondata'); - increasedAwaitDrain = false; - var ret = dest.write(chunk); - if (false === ret && !increasedAwaitDrain) { - // If the user unpiped during `dest.write()`, it is possible - // to get stuck in a permanently paused state if that write - // also returned false. - // => Check whether `dest` is still a piping destination. - if ((state.pipesCount === 1 && state.pipes === dest || state.pipesCount > 1 && indexOf(state.pipes, dest) !== -1) && !cleanedUp) { - debug('false write response, pause', src._readableState.awaitDrain); - src._readableState.awaitDrain++; - increasedAwaitDrain = true; - } - src.pause(); - } - } - - // if the dest has an error, then stop piping into it. - // however, don't suppress the throwing behavior for this. - function onerror(er) { - debug('onerror', er); - unpipe(); - dest.removeListener('error', onerror); - if (listenerCount(dest, 'error') === 0) dest.emit('error', er); - } - - // Make sure our error handler is attached before userland ones. - prependListener(dest, 'error', onerror); - - // Both close and finish should trigger unpipe, but only once. - function onclose() { - dest.removeListener('finish', onfinish); - unpipe(); - } - dest.once('close', onclose); - function onfinish() { - debug('onfinish'); - dest.removeListener('close', onclose); - unpipe(); - } - dest.once('finish', onfinish); - - function unpipe() { - debug('unpipe'); - src.unpipe(dest); - } - - // tell the dest that it's being piped to - dest.emit('pipe', src); - - // start the flow if it hasn't been started already. - if (!state.flowing) { - debug('pipe resume'); - src.resume(); - } - - return dest; - }; - - function pipeOnDrain(src) { - return function () { - var state = src._readableState; - debug('pipeOnDrain', state.awaitDrain); - if (state.awaitDrain) state.awaitDrain--; - if (state.awaitDrain === 0 && src.listeners('data').length) { - state.flowing = true; - flow(src); - } - }; - } - - Readable.prototype.unpipe = function (dest) { - var state = this._readableState; - - // if we're not piping anywhere, then do nothing. - if (state.pipesCount === 0) return this; - - // just one destination. most common case. - if (state.pipesCount === 1) { - // passed in one, but it's not the right one. - if (dest && dest !== state.pipes) return this; - - if (!dest) dest = state.pipes; - - // got a match. - state.pipes = null; - state.pipesCount = 0; - state.flowing = false; - if (dest) dest.emit('unpipe', this); - return this; - } - - // slow case. multiple pipe destinations. - - if (!dest) { - // remove all. - var dests = state.pipes; - var len = state.pipesCount; - state.pipes = null; - state.pipesCount = 0; - state.flowing = false; - - for (var _i = 0; _i < len; _i++) { - dests[_i].emit('unpipe', this); - }return this; - } - - // try to find the right one. - var i = indexOf(state.pipes, dest); - if (i === -1) return this; - - state.pipes.splice(i, 1); - state.pipesCount -= 1; - if (state.pipesCount === 1) state.pipes = state.pipes[0]; - - dest.emit('unpipe', this); - - return this; - }; - - // set up data events if they are asked for - // Ensure readable listeners eventually get something - Readable.prototype.on = function (ev, fn) { - var res = EventEmitter.prototype.on.call(this, ev, fn); - - if (ev === 'data') { - // Start flowing on next tick if stream isn't explicitly paused - if (this._readableState.flowing !== false) this.resume(); - } else if (ev === 'readable') { - var state = this._readableState; - if (!state.endEmitted && !state.readableListening) { - state.readableListening = state.needReadable = true; - state.emittedReadable = false; - if (!state.reading) { - nextTick(nReadingNextTick, this); - } else if (state.length) { - emitReadable(this); - } - } - } - - return res; - }; - Readable.prototype.addListener = Readable.prototype.on; - - function nReadingNextTick(self) { - debug('readable nexttick read 0'); - self.read(0); - } - - // pause() and resume() are remnants of the legacy readable stream API - // If the user uses them, then switch into old mode. - Readable.prototype.resume = function () { - var state = this._readableState; - if (!state.flowing) { - debug('resume'); - state.flowing = true; - resume(this, state); - } - return this; - }; - - function resume(stream, state) { - if (!state.resumeScheduled) { - state.resumeScheduled = true; - nextTick(resume_, stream, state); - } - } - - function resume_(stream, state) { - if (!state.reading) { - debug('resume read 0'); - stream.read(0); - } - - state.resumeScheduled = false; - state.awaitDrain = 0; - stream.emit('resume'); - flow(stream); - if (state.flowing && !state.reading) stream.read(0); - } - - Readable.prototype.pause = function () { - debug('call pause flowing=%j', this._readableState.flowing); - if (false !== this._readableState.flowing) { - debug('pause'); - this._readableState.flowing = false; - this.emit('pause'); - } - return this; - }; - - function flow(stream) { - var state = stream._readableState; - debug('flow', state.flowing); - while (state.flowing && stream.read() !== null) {} - } - - // wrap an old-style stream as the async data source. - // This is *not* part of the readable stream interface. - // It is an ugly unfortunate mess of history. - Readable.prototype.wrap = function (stream) { - var state = this._readableState; - var paused = false; - - var self = this; - stream.on('end', function () { - debug('wrapped end'); - if (state.decoder && !state.ended) { - var chunk = state.decoder.end(); - if (chunk && chunk.length) self.push(chunk); - } - - self.push(null); - }); - - stream.on('data', function (chunk) { - debug('wrapped data'); - if (state.decoder) chunk = state.decoder.write(chunk); - - // don't skip over falsy values in objectMode - if (state.objectMode && (chunk === null || chunk === undefined)) return;else if (!state.objectMode && (!chunk || !chunk.length)) return; - - var ret = self.push(chunk); - if (!ret) { - paused = true; - stream.pause(); - } - }); - - // proxy all the other methods. - // important when wrapping filters and duplexes. - for (var i in stream) { - if (this[i] === undefined && typeof stream[i] === 'function') { - this[i] = function (method) { - return function () { - return stream[method].apply(stream, arguments); - }; - }(i); - } - } - - // proxy certain important events. - var events = ['error', 'close', 'destroy', 'pause', 'resume']; - forEach(events, function (ev) { - stream.on(ev, self.emit.bind(self, ev)); - }); - - // when we try to consume some more bytes, simply unpause the - // underlying stream. - self._read = function (n) { - debug('wrapped _read', n); - if (paused) { - paused = false; - stream.resume(); - } - }; - - return self; - }; - - // exposed for testing purposes only. - Readable._fromList = fromList; - - // Pluck off n bytes from an array of buffers. - // Length is the combined lengths of all the buffers in the list. - // This function is designed to be inlinable, so please take care when making - // changes to the function body. - function fromList(n, state) { - // nothing buffered - if (state.length === 0) return null; - - var ret; - if (state.objectMode) ret = state.buffer.shift();else if (!n || n >= state.length) { - // read it all, truncate the list - if (state.decoder) ret = state.buffer.join('');else if (state.buffer.length === 1) ret = state.buffer.head.data;else ret = state.buffer.concat(state.length); - state.buffer.clear(); - } else { - // read part of list - ret = fromListPartial(n, state.buffer, state.decoder); - } - - return ret; - } - - // Extracts only enough buffered data to satisfy the amount requested. - // This function is designed to be inlinable, so please take care when making - // changes to the function body. - function fromListPartial(n, list, hasStrings) { - var ret; - if (n < list.head.data.length) { - // slice is the same for buffers and strings - ret = list.head.data.slice(0, n); - list.head.data = list.head.data.slice(n); - } else if (n === list.head.data.length) { - // first chunk is a perfect match - ret = list.shift(); - } else { - // result spans more than one buffer - ret = hasStrings ? copyFromBufferString(n, list) : copyFromBuffer(n, list); - } - return ret; - } - - // Copies a specified amount of characters from the list of buffered data - // chunks. - // This function is designed to be inlinable, so please take care when making - // changes to the function body. - function copyFromBufferString(n, list) { - var p = list.head; - var c = 1; - var ret = p.data; - n -= ret.length; - while (p = p.next) { - var str = p.data; - var nb = n > str.length ? str.length : n; - if (nb === str.length) ret += str;else ret += str.slice(0, n); - n -= nb; - if (n === 0) { - if (nb === str.length) { - ++c; - if (p.next) list.head = p.next;else list.head = list.tail = null; - } else { - list.head = p; - p.data = str.slice(nb); - } - break; - } - ++c; - } - list.length -= c; - return ret; - } - - // Copies a specified amount of bytes from the list of buffered data chunks. - // This function is designed to be inlinable, so please take care when making - // changes to the function body. - function copyFromBuffer(n, list) { - var ret = Buffer.allocUnsafe(n); - var p = list.head; - var c = 1; - p.data.copy(ret); - n -= p.data.length; - while (p = p.next) { - var buf = p.data; - var nb = n > buf.length ? buf.length : n; - buf.copy(ret, ret.length - n, 0, nb); - n -= nb; - if (n === 0) { - if (nb === buf.length) { - ++c; - if (p.next) list.head = p.next;else list.head = list.tail = null; - } else { - list.head = p; - p.data = buf.slice(nb); - } - break; - } - ++c; - } - list.length -= c; - return ret; - } - - function endReadable(stream) { - var state = stream._readableState; - - // If we get here before consuming all the bytes, then that is a - // bug in node. Should never happen. - if (state.length > 0) throw new Error('"endReadable()" called on non-empty stream'); - - if (!state.endEmitted) { - state.ended = true; - nextTick(endReadableNT, state, stream); - } - } - - function endReadableNT(state, stream) { - // Check that we didn't get one last unshift. - if (!state.endEmitted && state.length === 0) { - state.endEmitted = true; - stream.readable = false; - stream.emit('end'); - } - } - - function forEach(xs, f) { - for (var i = 0, l = xs.length; i < l; i++) { - f(xs[i], i); - } - } - - function indexOf(xs, x) { - for (var i = 0, l = xs.length; i < l; i++) { - if (xs[i] === x) return i; - } - return -1; - } - - // A bit simpler than readable streams. - Writable.WritableState = WritableState; - inherits$1(Writable, EventEmitter); - - function nop() {} - - function WriteReq(chunk, encoding, cb) { - this.chunk = chunk; - this.encoding = encoding; - this.callback = cb; - this.next = null; - } - - function WritableState(options, stream) { - Object.defineProperty(this, 'buffer', { - get: deprecate(function () { - return this.getBuffer(); - }, '_writableState.buffer is deprecated. Use _writableState.getBuffer ' + 'instead.') - }); - options = options || {}; - - // object stream flag to indicate whether or not this stream - // contains buffers or objects. - this.objectMode = !!options.objectMode; - - if (stream instanceof Duplex) this.objectMode = this.objectMode || !!options.writableObjectMode; - - // the point at which write() starts returning false - // Note: 0 is a valid value, means that we always return false if - // the entire buffer is not flushed immediately on write() - var hwm = options.highWaterMark; - var defaultHwm = this.objectMode ? 16 : 16 * 1024; - this.highWaterMark = hwm || hwm === 0 ? hwm : defaultHwm; - - // cast to ints. - this.highWaterMark = ~ ~this.highWaterMark; - - this.needDrain = false; - // at the start of calling end() - this.ending = false; - // when end() has been called, and returned - this.ended = false; - // when 'finish' is emitted - this.finished = false; - - // should we decode strings into buffers before passing to _write? - // this is here so that some node-core streams can optimize string - // handling at a lower level. - var noDecode = options.decodeStrings === false; - this.decodeStrings = !noDecode; - - // Crypto is kind of old and crusty. Historically, its default string - // encoding is 'binary' so we have to make this configurable. - // Everything else in the universe uses 'utf8', though. - this.defaultEncoding = options.defaultEncoding || 'utf8'; - - // not an actual buffer we keep track of, but a measurement - // of how much we're waiting to get pushed to some underlying - // socket or file. - this.length = 0; - - // a flag to see when we're in the middle of a write. - this.writing = false; - - // when true all writes will be buffered until .uncork() call - this.corked = 0; - - // a flag to be able to tell if the onwrite cb is called immediately, - // or on a later tick. We set this to true at first, because any - // actions that shouldn't happen until "later" should generally also - // not happen before the first write call. - this.sync = true; - - // a flag to know if we're processing previously buffered items, which - // may call the _write() callback in the same tick, so that we don't - // end up in an overlapped onwrite situation. - this.bufferProcessing = false; - - // the callback that's passed to _write(chunk,cb) - this.onwrite = function (er) { - onwrite(stream, er); - }; - - // the callback that the user supplies to write(chunk,encoding,cb) - this.writecb = null; - - // the amount that is being written when _write is called. - this.writelen = 0; - - this.bufferedRequest = null; - this.lastBufferedRequest = null; - - // number of pending user-supplied write callbacks - // this must be 0 before 'finish' can be emitted - this.pendingcb = 0; - - // emit prefinish if the only thing we're waiting for is _write cbs - // This is relevant for synchronous Transform streams - this.prefinished = false; - - // True if the error was already emitted and should not be thrown again - this.errorEmitted = false; - - // count buffered requests - this.bufferedRequestCount = 0; - - // allocate the first CorkedRequest, there is always - // one allocated and free to use, and we maintain at most two - this.corkedRequestsFree = new CorkedRequest(this); - } - - WritableState.prototype.getBuffer = function writableStateGetBuffer() { - var current = this.bufferedRequest; - var out = []; - while (current) { - out.push(current); - current = current.next; - } - return out; - }; - function Writable(options) { - - // Writable ctor is applied to Duplexes, though they're not - // instanceof Writable, they're instanceof Readable. - if (!(this instanceof Writable) && !(this instanceof Duplex)) return new Writable(options); - - this._writableState = new WritableState(options, this); - - // legacy. - this.writable = true; - - if (options) { - if (typeof options.write === 'function') this._write = options.write; - - if (typeof options.writev === 'function') this._writev = options.writev; - } - - EventEmitter.call(this); - } - - // Otherwise people can pipe Writable streams, which is just wrong. - Writable.prototype.pipe = function () { - this.emit('error', new Error('Cannot pipe, not readable')); - }; - - function writeAfterEnd(stream, cb) { - var er = new Error('write after end'); - // TODO: defer error events consistently everywhere, not just the cb - stream.emit('error', er); - nextTick(cb, er); - } - - // If we get something that is not a buffer, string, null, or undefined, - // and we're not in objectMode, then that's an error. - // Otherwise stream chunks are all considered to be of length=1, and the - // watermarks determine how many objects to keep in the buffer, rather than - // how many bytes or characters. - function validChunk(stream, state, chunk, cb) { - var valid = true; - var er = false; - // Always throw error if a null is written - // if we are not in object mode then throw - // if it is not a buffer, string, or undefined. - if (chunk === null) { - er = new TypeError('May not write null values to stream'); - } else if (!Buffer.isBuffer(chunk) && typeof chunk !== 'string' && chunk !== undefined && !state.objectMode) { - er = new TypeError('Invalid non-string/buffer chunk'); - } - if (er) { - stream.emit('error', er); - nextTick(cb, er); - valid = false; - } - return valid; - } - - Writable.prototype.write = function (chunk, encoding, cb) { - var state = this._writableState; - var ret = false; - - if (typeof encoding === 'function') { - cb = encoding; - encoding = null; - } - - if (Buffer.isBuffer(chunk)) encoding = 'buffer';else if (!encoding) encoding = state.defaultEncoding; - - if (typeof cb !== 'function') cb = nop; - - if (state.ended) writeAfterEnd(this, cb);else if (validChunk(this, state, chunk, cb)) { - state.pendingcb++; - ret = writeOrBuffer(this, state, chunk, encoding, cb); - } - - return ret; - }; - - Writable.prototype.cork = function () { - var state = this._writableState; - - state.corked++; - }; - - Writable.prototype.uncork = function () { - var state = this._writableState; - - if (state.corked) { - state.corked--; - - if (!state.writing && !state.corked && !state.finished && !state.bufferProcessing && state.bufferedRequest) clearBuffer(this, state); - } - }; - - Writable.prototype.setDefaultEncoding = function setDefaultEncoding(encoding) { - // node::ParseEncoding() requires lower case. - if (typeof encoding === 'string') encoding = encoding.toLowerCase(); - if (!(['hex', 'utf8', 'utf-8', 'ascii', 'binary', 'base64', 'ucs2', 'ucs-2', 'utf16le', 'utf-16le', 'raw'].indexOf((encoding + '').toLowerCase()) > -1)) throw new TypeError('Unknown encoding: ' + encoding); - this._writableState.defaultEncoding = encoding; + var domain; + + // This constructor is used to store event handlers. Instantiating this is + // faster than explicitly calling `Object.create(null)` to get a "clean" empty + // object (tested with v8 v4.9). + function EventHandlers() {} + EventHandlers.prototype = Object.create(null); + + function EventEmitter() { + EventEmitter.init.call(this); + } + + // nodejs oddity + // require('events') === require('events').EventEmitter + EventEmitter.EventEmitter = EventEmitter; + + EventEmitter.usingDomains = false; + + EventEmitter.prototype.domain = undefined; + EventEmitter.prototype._events = undefined; + EventEmitter.prototype._maxListeners = undefined; + + // By default EventEmitters will print a warning if more than 10 listeners are + // added to it. This is a useful default which helps finding memory leaks. + EventEmitter.defaultMaxListeners = 10; + + EventEmitter.init = function() { + this.domain = null; + if (EventEmitter.usingDomains) { + // if there is an active domain, then attach to it. + if (domain.active ) ; + } + + if (!this._events || this._events === Object.getPrototypeOf(this)._events) { + this._events = new EventHandlers(); + this._eventsCount = 0; + } + + this._maxListeners = this._maxListeners || undefined; + }; + + // Obviously not all Emitters should be limited to 10. This function allows + // that to be increased. Set to zero for unlimited. + EventEmitter.prototype.setMaxListeners = function setMaxListeners(n) { + if (typeof n !== 'number' || n < 0 || isNaN(n)) + throw new TypeError('"n" argument must be a positive number'); + this._maxListeners = n; + return this; + }; + + function $getMaxListeners(that) { + if (that._maxListeners === undefined) + return EventEmitter.defaultMaxListeners; + return that._maxListeners; + } + + EventEmitter.prototype.getMaxListeners = function getMaxListeners() { + return $getMaxListeners(this); + }; + + // These standalone emit* functions are used to optimize calling of event + // handlers for fast cases because emit() itself often has a variable number of + // arguments and can be deoptimized because of that. These functions always have + // the same number of arguments and thus do not get deoptimized, so the code + // inside them can execute faster. + function emitNone(handler, isFn, self) { + if (isFn) + handler.call(self); + else { + var len = handler.length; + var listeners = arrayClone(handler, len); + for (var i = 0; i < len; ++i) + listeners[i].call(self); + } + } + function emitOne(handler, isFn, self, arg1) { + if (isFn) + handler.call(self, arg1); + else { + var len = handler.length; + var listeners = arrayClone(handler, len); + for (var i = 0; i < len; ++i) + listeners[i].call(self, arg1); + } + } + function emitTwo(handler, isFn, self, arg1, arg2) { + if (isFn) + handler.call(self, arg1, arg2); + else { + var len = handler.length; + var listeners = arrayClone(handler, len); + for (var i = 0; i < len; ++i) + listeners[i].call(self, arg1, arg2); + } + } + function emitThree(handler, isFn, self, arg1, arg2, arg3) { + if (isFn) + handler.call(self, arg1, arg2, arg3); + else { + var len = handler.length; + var listeners = arrayClone(handler, len); + for (var i = 0; i < len; ++i) + listeners[i].call(self, arg1, arg2, arg3); + } + } + + function emitMany(handler, isFn, self, args) { + if (isFn) + handler.apply(self, args); + else { + var len = handler.length; + var listeners = arrayClone(handler, len); + for (var i = 0; i < len; ++i) + listeners[i].apply(self, args); + } + } + + EventEmitter.prototype.emit = function emit(type) { + var er, handler, len, args, i, events, domain; + var doError = (type === 'error'); + + events = this._events; + if (events) + doError = (doError && events.error == null); + else if (!doError) + return false; + + domain = this.domain; + + // If there is no 'error' event listener then throw. + if (doError) { + er = arguments[1]; + if (domain) { + if (!er) + er = new Error('Uncaught, unspecified "error" event'); + er.domainEmitter = this; + er.domain = domain; + er.domainThrown = false; + domain.emit('error', er); + } else if (er instanceof Error) { + throw er; // Unhandled 'error' event + } else { + // At least give some kind of context to the user + var err = new Error('Uncaught, unspecified "error" event. (' + er + ')'); + err.context = er; + throw err; + } + return false; + } + + handler = events[type]; + + if (!handler) + return false; + + var isFn = typeof handler === 'function'; + len = arguments.length; + switch (len) { + // fast cases + case 1: + emitNone(handler, isFn, this); + break; + case 2: + emitOne(handler, isFn, this, arguments[1]); + break; + case 3: + emitTwo(handler, isFn, this, arguments[1], arguments[2]); + break; + case 4: + emitThree(handler, isFn, this, arguments[1], arguments[2], arguments[3]); + break; + // slower + default: + args = new Array(len - 1); + for (i = 1; i < len; i++) + args[i - 1] = arguments[i]; + emitMany(handler, isFn, this, args); + } + + return true; + }; + + function _addListener(target, type, listener, prepend) { + var m; + var events; + var existing; + + if (typeof listener !== 'function') + throw new TypeError('"listener" argument must be a function'); + + events = target._events; + if (!events) { + events = target._events = new EventHandlers(); + target._eventsCount = 0; + } else { + // To avoid recursion in the case that type === "newListener"! Before + // adding it to the listeners, first emit "newListener". + if (events.newListener) { + target.emit('newListener', type, + listener.listener ? listener.listener : listener); + + // Re-assign `events` because a newListener handler could have caused the + // this._events to be assigned to a new object + events = target._events; + } + existing = events[type]; + } + + if (!existing) { + // Optimize the case of one listener. Don't need the extra array object. + existing = events[type] = listener; + ++target._eventsCount; + } else { + if (typeof existing === 'function') { + // Adding the second element, need to change to array. + existing = events[type] = prepend ? [listener, existing] : + [existing, listener]; + } else { + // If we've already got an array, just append. + if (prepend) { + existing.unshift(listener); + } else { + existing.push(listener); + } + } + + // Check for listener leak + if (!existing.warned) { + m = $getMaxListeners(target); + if (m && m > 0 && existing.length > m) { + existing.warned = true; + var w = new Error('Possible EventEmitter memory leak detected. ' + + existing.length + ' ' + type + ' listeners added. ' + + 'Use emitter.setMaxListeners() to increase limit'); + w.name = 'MaxListenersExceededWarning'; + w.emitter = target; + w.type = type; + w.count = existing.length; + emitWarning(w); + } + } + } + + return target; + } + function emitWarning(e) { + typeof console.warn === 'function' ? console.warn(e) : console.log(e); + } + EventEmitter.prototype.addListener = function addListener(type, listener) { + return _addListener(this, type, listener, false); + }; + + EventEmitter.prototype.on = EventEmitter.prototype.addListener; + + EventEmitter.prototype.prependListener = + function prependListener(type, listener) { + return _addListener(this, type, listener, true); + }; + + function _onceWrap(target, type, listener) { + var fired = false; + function g() { + target.removeListener(type, g); + if (!fired) { + fired = true; + listener.apply(target, arguments); + } + } + g.listener = listener; + return g; + } + + EventEmitter.prototype.once = function once(type, listener) { + if (typeof listener !== 'function') + throw new TypeError('"listener" argument must be a function'); + this.on(type, _onceWrap(this, type, listener)); + return this; + }; + + EventEmitter.prototype.prependOnceListener = + function prependOnceListener(type, listener) { + if (typeof listener !== 'function') + throw new TypeError('"listener" argument must be a function'); + this.prependListener(type, _onceWrap(this, type, listener)); + return this; + }; + + // emits a 'removeListener' event iff the listener was removed + EventEmitter.prototype.removeListener = + function removeListener(type, listener) { + var list, events, position, i, originalListener; + + if (typeof listener !== 'function') + throw new TypeError('"listener" argument must be a function'); + + events = this._events; + if (!events) + return this; + + list = events[type]; + if (!list) + return this; + + if (list === listener || (list.listener && list.listener === listener)) { + if (--this._eventsCount === 0) + this._events = new EventHandlers(); + else { + delete events[type]; + if (events.removeListener) + this.emit('removeListener', type, list.listener || listener); + } + } else if (typeof list !== 'function') { + position = -1; + + for (i = list.length; i-- > 0;) { + if (list[i] === listener || + (list[i].listener && list[i].listener === listener)) { + originalListener = list[i].listener; + position = i; + break; + } + } + + if (position < 0) + return this; + + if (list.length === 1) { + list[0] = undefined; + if (--this._eventsCount === 0) { + this._events = new EventHandlers(); return this; - }; - - function decodeChunk(state, chunk, encoding) { - if (!state.objectMode && state.decodeStrings !== false && typeof chunk === 'string') { - chunk = Buffer.from(chunk, encoding); - } - return chunk; - } - - // if we're already writing something, then just put this - // in the queue, and wait our turn. Otherwise, call _write - // If we return false, then we need a drain event, so set that flag. - function writeOrBuffer(stream, state, chunk, encoding, cb) { - chunk = decodeChunk(state, chunk, encoding); - - if (Buffer.isBuffer(chunk)) encoding = 'buffer'; - var len = state.objectMode ? 1 : chunk.length; - - state.length += len; - - var ret = state.length < state.highWaterMark; - // we must ensure that previous needDrain will not be reset to false. - if (!ret) state.needDrain = true; - - if (state.writing || state.corked) { - var last = state.lastBufferedRequest; - state.lastBufferedRequest = new WriteReq(chunk, encoding, cb); - if (last) { - last.next = state.lastBufferedRequest; - } else { - state.bufferedRequest = state.lastBufferedRequest; - } - state.bufferedRequestCount += 1; - } else { - doWrite(stream, state, false, len, chunk, encoding, cb); - } - - return ret; - } - - function doWrite(stream, state, writev, len, chunk, encoding, cb) { - state.writelen = len; - state.writecb = cb; - state.writing = true; - state.sync = true; - if (writev) stream._writev(chunk, state.onwrite);else stream._write(chunk, encoding, state.onwrite); - state.sync = false; - } - - function onwriteError(stream, state, sync, er, cb) { - --state.pendingcb; - if (sync) nextTick(cb, er);else cb(er); - - stream._writableState.errorEmitted = true; - stream.emit('error', er); - } - - function onwriteStateUpdate(state) { - state.writing = false; - state.writecb = null; - state.length -= state.writelen; - state.writelen = 0; - } - - function onwrite(stream, er) { - var state = stream._writableState; - var sync = state.sync; - var cb = state.writecb; - - onwriteStateUpdate(state); - - if (er) onwriteError(stream, state, sync, er, cb);else { - // Check if we're actually ready to finish, but don't emit yet - var finished = needFinish(state); - - if (!finished && !state.corked && !state.bufferProcessing && state.bufferedRequest) { - clearBuffer(stream, state); - } - - if (sync) { - /**/ - nextTick(afterWrite, stream, state, finished, cb); - /**/ - } else { - afterWrite(stream, state, finished, cb); - } - } - } - - function afterWrite(stream, state, finished, cb) { - if (!finished) onwriteDrain(stream, state); - state.pendingcb--; - cb(); - finishMaybe(stream, state); - } - - // Must force callback to be called on nextTick, so that we don't - // emit 'drain' before the write() consumer gets the 'false' return - // value, and has a chance to attach a 'drain' listener. - function onwriteDrain(stream, state) { - if (state.length === 0 && state.needDrain) { - state.needDrain = false; - stream.emit('drain'); - } - } - - // if there's something in the buffer waiting, then process it - function clearBuffer(stream, state) { - state.bufferProcessing = true; - var entry = state.bufferedRequest; - - if (stream._writev && entry && entry.next) { - // Fast case, write everything using _writev() - var l = state.bufferedRequestCount; - var buffer = new Array(l); - var holder = state.corkedRequestsFree; - holder.entry = entry; - - var count = 0; - while (entry) { - buffer[count] = entry; - entry = entry.next; - count += 1; - } - - doWrite(stream, state, true, state.length, buffer, '', holder.finish); - - // doWrite is almost always async, defer these to save a bit of time - // as the hot path ends with doWrite - state.pendingcb++; - state.lastBufferedRequest = null; - if (holder.next) { - state.corkedRequestsFree = holder.next; - holder.next = null; - } else { - state.corkedRequestsFree = new CorkedRequest(state); - } - } else { - // Slow case, write chunks one-by-one - while (entry) { - var chunk = entry.chunk; - var encoding = entry.encoding; - var cb = entry.callback; - var len = state.objectMode ? 1 : chunk.length; - - doWrite(stream, state, false, len, chunk, encoding, cb); - entry = entry.next; - // if we didn't call the onwrite immediately, then - // it means that we need to wait until it does. - // also, that means that the chunk and cb are currently - // being processed, so move the buffer counter past them. - if (state.writing) { - break; - } - } - - if (entry === null) state.lastBufferedRequest = null; - } - - state.bufferedRequestCount = 0; - state.bufferedRequest = entry; - state.bufferProcessing = false; - } - - Writable.prototype._write = function (chunk, encoding, cb) { - cb(new Error('not implemented')); - }; - - Writable.prototype._writev = null; - - Writable.prototype.end = function (chunk, encoding, cb) { - var state = this._writableState; - - if (typeof chunk === 'function') { - cb = chunk; - chunk = null; - encoding = null; - } else if (typeof encoding === 'function') { - cb = encoding; - encoding = null; - } - - if (chunk !== null && chunk !== undefined) this.write(chunk, encoding); - - // .end() fully uncorks - if (state.corked) { - state.corked = 1; - this.uncork(); - } - - // ignore unnecessary end() calls. - if (!state.ending && !state.finished) endWritable(this, state, cb); - }; - - function needFinish(state) { - return state.ending && state.length === 0 && state.bufferedRequest === null && !state.finished && !state.writing; - } - - function prefinish(stream, state) { - if (!state.prefinished) { - state.prefinished = true; - stream.emit('prefinish'); - } - } - - function finishMaybe(stream, state) { - var need = needFinish(state); - if (need) { - if (state.pendingcb === 0) { - prefinish(stream, state); - state.finished = true; - stream.emit('finish'); - } else { - prefinish(stream, state); - } - } - return need; - } - - function endWritable(stream, state, cb) { - state.ending = true; - finishMaybe(stream, state); - if (cb) { - if (state.finished) nextTick(cb);else stream.once('finish', cb); - } - state.ended = true; - stream.writable = false; - } - - // It seems a linked list but it is not - // there will be only 2 of these for each stream - function CorkedRequest(state) { - var _this = this; - - this.next = null; - this.entry = null; - - this.finish = function (err) { - var entry = _this.entry; - _this.entry = null; - while (entry) { - var cb = entry.callback; - state.pendingcb--; - cb(err); - entry = entry.next; - } - if (state.corkedRequestsFree) { - state.corkedRequestsFree.next = _this; - } else { - state.corkedRequestsFree = _this; - } - }; - } - - inherits$1(Duplex, Readable); - - var keys = Object.keys(Writable.prototype); - for (var v = 0; v < keys.length; v++) { - var method = keys[v]; - if (!Duplex.prototype[method]) Duplex.prototype[method] = Writable.prototype[method]; - } - function Duplex(options) { - if (!(this instanceof Duplex)) return new Duplex(options); - - Readable.call(this, options); - Writable.call(this, options); - - if (options && options.readable === false) this.readable = false; - - if (options && options.writable === false) this.writable = false; - - this.allowHalfOpen = true; - if (options && options.allowHalfOpen === false) this.allowHalfOpen = false; - - this.once('end', onend); - } - - // the no-half-open enforcer - function onend() { - // if we allow half-open state, or if the writable side ended, - // then we're ok. - if (this.allowHalfOpen || this._writableState.ended) return; - - // no more data can be written. - // But allow more writes to happen in this tick. - nextTick(onEndNT, this); - } - - function onEndNT(self) { - self.end(); - } - - // a transform stream is a readable/writable stream where you do - inherits$1(Transform, Duplex); - - function TransformState(stream) { - this.afterTransform = function (er, data) { - return afterTransform(stream, er, data); - }; - - this.needTransform = false; - this.transforming = false; - this.writecb = null; - this.writechunk = null; - this.writeencoding = null; - } - - function afterTransform(stream, er, data) { - var ts = stream._transformState; - ts.transforming = false; - - var cb = ts.writecb; - - if (!cb) return stream.emit('error', new Error('no writecb in Transform class')); - - ts.writechunk = null; - ts.writecb = null; - - if (data !== null && data !== undefined) stream.push(data); - - cb(er); - - var rs = stream._readableState; - rs.reading = false; - if (rs.needReadable || rs.length < rs.highWaterMark) { - stream._read(rs.highWaterMark); - } - } - function Transform(options) { - if (!(this instanceof Transform)) return new Transform(options); - - Duplex.call(this, options); - - this._transformState = new TransformState(this); - - // when the writable side finishes, then flush out anything remaining. - var stream = this; - - // start out asking for a readable event once data is transformed. - this._readableState.needReadable = true; - - // we have implemented the _read method, and done the other things - // that Readable wants before the first _read call, so unset the - // sync guard flag. - this._readableState.sync = false; - - if (options) { - if (typeof options.transform === 'function') this._transform = options.transform; - - if (typeof options.flush === 'function') this._flush = options.flush; - } - - this.once('prefinish', function () { - if (typeof this._flush === 'function') this._flush(function (er) { - done(stream, er); - });else done(stream); - }); - } - - Transform.prototype.push = function (chunk, encoding) { - this._transformState.needTransform = false; - return Duplex.prototype.push.call(this, chunk, encoding); - }; - - // This is the part where you do stuff! - // override this function in implementation classes. - // 'chunk' is an input chunk. - // - // Call `push(newChunk)` to pass along transformed output - // to the readable side. You may call 'push' zero or more times. - // - // Call `cb(err)` when you are done with this chunk. If you pass - // an error, then that'll put the hurt on the whole operation. If you - // never call cb(), then you'll never get another chunk. - Transform.prototype._transform = function (chunk, encoding, cb) { - throw new Error('Not implemented'); - }; - - Transform.prototype._write = function (chunk, encoding, cb) { - var ts = this._transformState; - ts.writecb = cb; - ts.writechunk = chunk; - ts.writeencoding = encoding; - if (!ts.transforming) { - var rs = this._readableState; - if (ts.needTransform || rs.needReadable || rs.length < rs.highWaterMark) this._read(rs.highWaterMark); - } - }; - - // Doesn't matter what the args are here. - // _transform does all the work. - // That we got here means that the readable side wants more data. - Transform.prototype._read = function (n) { - var ts = this._transformState; - - if (ts.writechunk !== null && ts.writecb && !ts.transforming) { - ts.transforming = true; - this._transform(ts.writechunk, ts.writeencoding, ts.afterTransform); - } else { - // mark that we need a transform, so that any data that comes in - // will get processed, now that we've asked for it. - ts.needTransform = true; - } - }; - - function done(stream, er) { - if (er) return stream.emit('error', er); - - // if there's nothing in the write buffer, then that means - // that nothing more will ever be provided - var ws = stream._writableState; - var ts = stream._transformState; - - if (ws.length) throw new Error('Calling transform done when ws.length != 0'); - - if (ts.transforming) throw new Error('Calling transform done when still transforming'); - - return stream.push(null); - } - - inherits$1(PassThrough, Transform); - function PassThrough(options) { - if (!(this instanceof PassThrough)) return new PassThrough(options); - - Transform.call(this, options); - } - - PassThrough.prototype._transform = function (chunk, encoding, cb) { - cb(null, chunk); - }; - - inherits$1(Stream, EventEmitter); - Stream.Readable = Readable; - Stream.Writable = Writable; - Stream.Duplex = Duplex; - Stream.Transform = Transform; - Stream.PassThrough = PassThrough; - - // Backwards-compat with node 0.4.x - Stream.Stream = Stream; - - // old-style streams. Note that the pipe method (the only relevant - // part of this class) is overridden in the Readable class. - - function Stream() { - EventEmitter.call(this); - } - - Stream.prototype.pipe = function(dest, options) { - var source = this; - - function ondata(chunk) { - if (dest.writable) { - if (false === dest.write(chunk) && source.pause) { - source.pause(); - } - } - } - - source.on('data', ondata); - - function ondrain() { - if (source.readable && source.resume) { - source.resume(); - } - } - - dest.on('drain', ondrain); - - // If the 'end' option is not supplied, dest.end() will be called when - // source gets the 'end' or 'close' events. Only dest.end() once. - if (!dest._isStdio && (!options || options.end !== false)) { - source.on('end', onend); - source.on('close', onclose); - } - - var didOnEnd = false; - function onend() { - if (didOnEnd) return; - didOnEnd = true; - - dest.end(); - } - - - function onclose() { - if (didOnEnd) return; - didOnEnd = true; - - if (typeof dest.destroy === 'function') dest.destroy(); - } - - // don't leave dangling pipes when there are errors. - function onerror(er) { - cleanup(); - if (EventEmitter.listenerCount(this, 'error') === 0) { - throw er; // Unhandled stream error in pipe. - } - } - - source.on('error', onerror); - dest.on('error', onerror); - - // remove all the event listeners that were added. - function cleanup() { - source.removeListener('data', ondata); - dest.removeListener('drain', ondrain); - - source.removeListener('end', onend); - source.removeListener('close', onclose); - - source.removeListener('error', onerror); - dest.removeListener('error', onerror); - - source.removeListener('end', cleanup); - source.removeListener('close', cleanup); - - dest.removeListener('close', cleanup); - } - - source.on('end', cleanup); - source.on('close', cleanup); - - dest.on('close', cleanup); - - dest.emit('pipe', source); - - // Allow for unix-like usage: A.pipe(B).pipe(C) - return dest; - }; - - const bom_utf8 = Buffer.from([239, 187, 191]); - - class CsvError extends Error { - constructor(code, message, ...contexts) { - if(Array.isArray(message)) message = message.join(' '); - super(message); - if(Error.captureStackTrace !== undefined){ - Error.captureStackTrace(this, CsvError); - } - this.code = code; - for(const context of contexts){ - for(const key in context){ - const value = context[key]; - this[key] = isBuffer(value) ? value.toString() : value == null ? value : JSON.parse(JSON.stringify(value)); - } - } - } - } - - const isObject = function(obj){ - return typeof obj === 'object' && obj !== null && ! Array.isArray(obj); - }; - - const underscore = function(str){ - return str.replace(/([A-Z])/g, function(_, match){ - return '_' + match.toLowerCase(); - }); - }; - - // Lodash implementation of `get` - - const charCodeOfDot = '.'.charCodeAt(0); - const reEscapeChar = /\\(\\)?/g; - const rePropName = RegExp( - // Match anything that isn't a dot or bracket. - '[^.[\\]]+' + '|' + - // Or match property names within brackets. - '\\[(?:' + - // Match a non-string expression. - '([^"\'][^[]*)' + '|' + - // Or match strings (supports escaping characters). - '(["\'])((?:(?!\\2)[^\\\\]|\\\\.)*?)\\2' + - ')\\]'+ '|' + - // Or match "" as the space between consecutive dots or empty brackets. - '(?=(?:\\.|\\[\\])(?:\\.|\\[\\]|$))' - , 'g'); - const reIsDeepProp = /\.|\[(?:[^[\]]*|(["'])(?:(?!\1)[^\\]|\\.)*?\1)\]/; - const reIsPlainProp = /^\w*$/; - const getTag = function(value){ - return Object.prototype.toString.call(value); - }; - const isSymbol = function(value){ - const type = typeof value; - return type === 'symbol' || (type === 'object' && value && getTag(value) === '[object Symbol]'); - }; - const isKey = function(value, object){ - if(Array.isArray(value)){ - return false; - } - const type = typeof value; - if(type === 'number' || type === 'symbol' || type === 'boolean' || !value || isSymbol(value)){ - return true; - } - return reIsPlainProp.test(value) || !reIsDeepProp.test(value) || - (object != null && value in Object(object)); - }; - const stringToPath = function(string){ - const result = []; - if(string.charCodeAt(0) === charCodeOfDot){ - result.push(''); - } - string.replace(rePropName, function(match, expression, quote, subString){ - let key = match; - if(quote){ - key = subString.replace(reEscapeChar, '$1'); - }else if(expression){ - key = expression.trim(); - } - result.push(key); - }); - return result; - }; - const castPath = function(value, object){ - if(Array.isArray(value)){ - return value; - } else { - return isKey(value, object) ? [value] : stringToPath(value); - } - }; - const toKey = function(value){ - if(typeof value === 'string' || isSymbol(value)) - return value; - const result = `${value}`; - // eslint-disable-next-line - return (result == '0' && (1 / value) == -INFINITY) ? '-0' : result; - }; - const get = function(object, path){ - path = castPath(path, object); - let index = 0; - const length = path.length; - while(object != null && index < length){ - object = object[toKey(path[index++])]; - } - return (index && index === length) ? object : undefined; - }; - - class Stringifier extends Transform { - constructor(opts = {}){ - super({...{writableObjectMode: true}, ...opts}); - const options = {}; - let err; - // Merge with user options - for(const opt in opts){ - options[underscore(opt)] = opts[opt]; - } - if((err = this.normalize(options)) !== undefined) throw err; - switch(options.record_delimiter){ - case 'auto': - options.record_delimiter = null; - break; - case 'unix': - options.record_delimiter = "\n"; - break; - case 'mac': - options.record_delimiter = "\r"; - break; - case 'windows': - options.record_delimiter = "\r\n"; - break; - case 'ascii': - options.record_delimiter = "\u001e"; - break; - case 'unicode': - options.record_delimiter = "\u2028"; - break; - } - // Expose options - this.options = options; - // Internal state - this.state = { - stop: false - }; - // Information - this.info = { - records: 0 - }; - } - normalize(options){ - // Normalize option `bom` - if(options.bom === undefined || options.bom === null || options.bom === false){ - options.bom = false; - }else if(options.bom !== true){ - return new CsvError('CSV_OPTION_BOOLEAN_INVALID_TYPE', [ - 'option `bom` is optional and must be a boolean value,', - `got ${JSON.stringify(options.bom)}` - ]); - } - // Normalize option `delimiter` - if(options.delimiter === undefined || options.delimiter === null){ - options.delimiter = ','; - }else if(isBuffer(options.delimiter)){ - options.delimiter = options.delimiter.toString(); - }else if(typeof options.delimiter !== 'string'){ - return new CsvError('CSV_OPTION_DELIMITER_INVALID_TYPE', [ - 'option `delimiter` must be a buffer or a string,', - `got ${JSON.stringify(options.delimiter)}` - ]); - } - // Normalize option `quote` - if(options.quote === undefined || options.quote === null){ - options.quote = '"'; - }else if(options.quote === true){ - options.quote = '"'; - }else if(options.quote === false){ - options.quote = ''; - }else if (isBuffer(options.quote)){ - options.quote = options.quote.toString(); - }else if(typeof options.quote !== 'string'){ - return new CsvError('CSV_OPTION_QUOTE_INVALID_TYPE', [ - 'option `quote` must be a boolean, a buffer or a string,', - `got ${JSON.stringify(options.quote)}` - ]); - } - // Normalize option `quoted` - if(options.quoted === undefined || options.quoted === null){ - options.quoted = false; - } - // Normalize option `quoted_empty` - if(options.quoted_empty === undefined || options.quoted_empty === null){ - options.quoted_empty = undefined; - } - // Normalize option `quoted_match` - if(options.quoted_match === undefined || options.quoted_match === null || options.quoted_match === false){ - options.quoted_match = null; - }else if(!Array.isArray(options.quoted_match)){ - options.quoted_match = [options.quoted_match]; - } - if(options.quoted_match){ - for(const quoted_match of options.quoted_match){ - const isString = typeof quoted_match === 'string'; - const isRegExp = quoted_match instanceof RegExp; - if(!isString && !isRegExp){ - return Error(`Invalid Option: quoted_match must be a string or a regex, got ${JSON.stringify(quoted_match)}`); - } - } - } - // Normalize option `quoted_string` - if(options.quoted_string === undefined || options.quoted_string === null){ - options.quoted_string = false; - } - // Normalize option `eof` - if(options.eof === undefined || options.eof === null){ - options.eof = true; - } - // Normalize option `escape` - if(options.escape === undefined || options.escape === null){ - options.escape = '"'; - }else if(isBuffer(options.escape)){ - options.escape = options.escape.toString(); - }else if(typeof options.escape !== 'string'){ - return Error(`Invalid Option: escape must be a buffer or a string, got ${JSON.stringify(options.escape)}`); - } - if (options.escape.length > 1){ - return Error(`Invalid Option: escape must be one character, got ${options.escape.length} characters`); - } - // Normalize option `header` - if(options.header === undefined || options.header === null){ - options.header = false; - } - // Normalize option `columns` - const [err, columns] = this.normalize_columns(options.columns); - if(err) return err; - options.columns = columns; - // Normalize option `quoted` - if(options.quoted === undefined || options.quoted === null){ - options.quoted = false; - } - // Normalize option `cast` - if(options.cast === undefined || options.cast === null){ - options.cast = {}; - } - // Normalize option cast.bigint - if(options.cast.bigint === undefined || options.cast.bigint === null){ - // Cast boolean to string by default - options.cast.bigint = value => '' + value; - } - // Normalize option cast.boolean - if(options.cast.boolean === undefined || options.cast.boolean === null){ - // Cast boolean to string by default - options.cast.boolean = value => value ? '1' : ''; - } - // Normalize option cast.date - if(options.cast.date === undefined || options.cast.date === null){ - // Cast date to timestamp string by default - options.cast.date = value => '' + value.getTime(); - } - // Normalize option cast.number - if(options.cast.number === undefined || options.cast.number === null){ - // Cast number to string using native casting by default - options.cast.number = value => '' + value; - } - // Normalize option cast.object - if(options.cast.object === undefined || options.cast.object === null){ - // Stringify object as JSON by default - options.cast.object = value => JSON.stringify(value); - } - // Normalize option cast.string - if(options.cast.string === undefined || options.cast.string === null){ - // Leave string untouched - options.cast.string = function(value){return value;}; - } - // Normalize option `record_delimiter` - if(options.record_delimiter === undefined || options.record_delimiter === null){ - options.record_delimiter = '\n'; - }else if(isBuffer(options.record_delimiter)){ - options.record_delimiter = options.record_delimiter.toString(); - }else if(typeof options.record_delimiter !== 'string'){ - return Error(`Invalid Option: record_delimiter must be a buffer or a string, got ${JSON.stringify(options.record_delimiter)}`); - } - } - _transform(chunk, encoding, callback){ - if(this.state.stop === true){ - return; - } - const err = this.__transform(chunk); - if(err !== undefined){ - this.state.stop = true; - } - callback(err); - } - _flush(callback){ - if(this.state.stop === true){ - // Note, Node.js 12 call flush even after an error, we must prevent - // `callback` from being called in flush without any error. - return; - } - if(this.info.records === 0){ - this.bom(); - const err = this.headers(); - if(err) callback(err); - } - callback(); - } - __transform(chunk){ - // Chunk validation - if(!Array.isArray(chunk) && typeof chunk !== 'object'){ - return Error(`Invalid Record: expect an array or an object, got ${JSON.stringify(chunk)}`); - } - // Detect columns from the first record - if(this.info.records === 0){ - if(Array.isArray(chunk)){ - if(this.options.header === true && this.options.columns === undefined){ - return Error('Undiscoverable Columns: header option requires column option or object records'); - } - }else if(this.options.columns === undefined){ - const [err, columns] = this.normalize_columns(Object.keys(chunk)); - if(err) return; - this.options.columns = columns; - } - } - // Emit the header - if(this.info.records === 0){ - this.bom(); - const err = this.headers(); - if(err) return err; - } - // Emit and stringify the record if an object or an array - try{ - this.emit('record', chunk, this.info.records); - }catch(err){ - return err; - } - // Convert the record into a string - let err, chunk_string; - if(this.options.eof){ - [err, chunk_string] = this.stringify(chunk); - if(err) return err; - if(chunk_string === undefined){ - return; - }else { - chunk_string = chunk_string + this.options.record_delimiter; - } - }else { - [err, chunk_string] = this.stringify(chunk); - if(err) return err; - if(chunk_string === undefined){ - return; - }else { - if(this.options.header || this.info.records){ - chunk_string = this.options.record_delimiter + chunk_string; - } - } - } - // Emit the csv - this.info.records++; - this.push(chunk_string); - } - stringify(chunk, chunkIsHeader=false){ - if(typeof chunk !== 'object'){ - return [undefined, chunk]; - } - const {columns} = this.options; - const record = []; - // Record is an array - if(Array.isArray(chunk)){ - // We are getting an array but the user has specified output columns. In - // this case, we respect the columns indexes - if(columns){ - chunk.splice(columns.length); - } - // Cast record elements - for(let i=0; i= 0; - const containsQuote = (quote !== '') && value.indexOf(quote) >= 0; - const containsEscape = value.indexOf(escape) >= 0 && (escape !== quote); - const containsRecordDelimiter = value.indexOf(record_delimiter) >= 0; - const quotedString = quoted_string && typeof field === 'string'; - let quotedMatch = quoted_match && quoted_match.filter(quoted_match => { - if(typeof quoted_match === 'string'){ - return value.indexOf(quoted_match) !== -1; - }else { - return quoted_match.test(value); - } - }); - quotedMatch = quotedMatch && quotedMatch.length > 0; - const shouldQuote = containsQuote === true || containsdelimiter || containsRecordDelimiter || quoted || quotedString || quotedMatch; - if(shouldQuote === true && containsEscape === true){ - const regexp = escape === '\\' - ? new RegExp(escape + escape, 'g') - : new RegExp(escape, 'g'); - value = value.replace(regexp, escape + escape); - } - if(containsQuote === true){ - const regexp = new RegExp(quote,'g'); - value = value.replace(regexp, escape + quote); - } - if(shouldQuote === true){ - value = quote + value + quote; - } - csvrecord += value; - }else if(quoted_empty === true || (field === '' && quoted_string === true && quoted_empty !== false)){ - csvrecord += quote + quote; - } - if(i !== record.length - 1){ - csvrecord += delimiter; - } - } - return [undefined, csvrecord]; - } - bom(){ - if(this.options.bom !== true){ - return; - } - this.push(bom_utf8); - } - headers(){ - if(this.options.header === false){ - return; - } - if(this.options.columns === undefined){ - return; - } - let err; - let headers = this.options.columns.map(column => column.header); - if(this.options.eof){ - [err, headers] = this.stringify(headers, true); - headers += this.options.record_delimiter; - }else { - [err, headers] = this.stringify(headers); - } - if(err) return err; - this.push(headers); - } - __cast(value, context){ - const type = typeof value; - try{ - if(type === 'string'){ // Fine for 99% of the cases - return [undefined, this.options.cast.string(value, context)]; - }else if(type === 'bigint'){ - return [undefined, this.options.cast.bigint(value, context)]; - }else if(type === 'number'){ - return [undefined, this.options.cast.number(value, context)]; - }else if(type === 'boolean'){ - return [undefined, this.options.cast.boolean(value, context)]; - }else if(value instanceof Date){ - return [undefined, this.options.cast.date(value, context)]; - }else if(type === 'object' && value !== null){ - return [undefined, this.options.cast.object(value, context)]; - }else { - return [undefined, value, value]; - } - }catch(err){ - return [err]; - } - } - normalize_columns(columns){ - if(columns === undefined || columns === null){ - return []; - } - if(typeof columns !== 'object'){ - return [Error('Invalid option "columns": expect an array or an object')]; - } - if(!Array.isArray(columns)){ - const newcolumns = []; - for(const k in columns){ - newcolumns.push({ - key: k, - header: columns[k] - }); - } - columns = newcolumns; - }else { - const newcolumns = []; - for(const column of columns){ - if(typeof column === 'string'){ - newcolumns.push({ - key: column, - header: column - }); - }else if(typeof column === 'object' && column !== undefined && !Array.isArray(column)){ - if(!column.key){ - return [Error('Invalid column definition: property "key" is required')]; - } - if(column.header === undefined){ - column.header = column.key; - } - newcolumns.push(column); - }else { - return [Error('Invalid column definition: expect a string or an object')]; - } - } - columns = newcolumns; - } - return [undefined, columns]; - } - } - - const stringify = function(){ - let data, options, callback; - for(const i in arguments){ - const argument = arguments[i]; - const type = typeof argument; - if(data === undefined && (Array.isArray(argument))){ - data = argument; - }else if(options === undefined && isObject(argument)){ - options = argument; - }else if(callback === undefined && type === 'function'){ - callback = argument; - }else { - throw new CsvError('CSV_INVALID_ARGUMENT', [ - 'Invalid argument:', - `got ${JSON.stringify(argument)} at index ${i}` - ]); - } - } - const stringifier = new Stringifier(options); - if(callback){ - const chunks = []; - stringifier.on('readable', function(){ - let chunk; - while((chunk = this.read()) !== null){ - chunks.push(chunk); - } - }); - stringifier.on('error', function(err){ - callback(err); - }); - stringifier.on('end', function(){ - callback(undefined, chunks.join('')); - }); - } - if(data !== undefined){ - const writer = function(){ - for(const record of data){ - stringifier.write(record); - } - stringifier.end(); - }; - // Support Deno, Rollup doesnt provide a shim for setImmediate - if(typeof setImmediate === 'function'){ - setImmediate(writer); - }else { - setTimeout(writer, 0); - } - } - return stringifier; - }; - - exports.CsvError = CsvError; - exports.Stringifier = Stringifier; - exports.stringify = stringify; - - Object.defineProperty(exports, '__esModule', { value: true }); + } else { + delete events[type]; + } + } else { + spliceOne(list, position); + } + + if (events.removeListener) + this.emit('removeListener', type, originalListener || listener); + } + + return this; + }; + + EventEmitter.prototype.removeAllListeners = + function removeAllListeners(type) { + var listeners, events; + + events = this._events; + if (!events) + return this; + + // not listening for removeListener, no need to emit + if (!events.removeListener) { + if (arguments.length === 0) { + this._events = new EventHandlers(); + this._eventsCount = 0; + } else if (events[type]) { + if (--this._eventsCount === 0) + this._events = new EventHandlers(); + else + delete events[type]; + } + return this; + } + + // emit removeListener for all listeners on all events + if (arguments.length === 0) { + var keys = Object.keys(events); + for (var i = 0, key; i < keys.length; ++i) { + key = keys[i]; + if (key === 'removeListener') continue; + this.removeAllListeners(key); + } + this.removeAllListeners('removeListener'); + this._events = new EventHandlers(); + this._eventsCount = 0; + return this; + } + + listeners = events[type]; + + if (typeof listeners === 'function') { + this.removeListener(type, listeners); + } else if (listeners) { + // LIFO order + do { + this.removeListener(type, listeners[listeners.length - 1]); + } while (listeners[0]); + } + + return this; + }; + + EventEmitter.prototype.listeners = function listeners(type) { + var evlistener; + var ret; + var events = this._events; + + if (!events) + ret = []; + else { + evlistener = events[type]; + if (!evlistener) + ret = []; + else if (typeof evlistener === 'function') + ret = [evlistener.listener || evlistener]; + else + ret = unwrapListeners(evlistener); + } + + return ret; + }; + + EventEmitter.listenerCount = function(emitter, type) { + if (typeof emitter.listenerCount === 'function') { + return emitter.listenerCount(type); + } else { + return listenerCount$1.call(emitter, type); + } + }; + + EventEmitter.prototype.listenerCount = listenerCount$1; + function listenerCount$1(type) { + var events = this._events; + + if (events) { + var evlistener = events[type]; + + if (typeof evlistener === 'function') { + return 1; + } else if (evlistener) { + return evlistener.length; + } + } + + return 0; + } + + EventEmitter.prototype.eventNames = function eventNames() { + return this._eventsCount > 0 ? Reflect.ownKeys(this._events) : []; + }; + + // About 1.5x faster than the two-arg version of Array#splice(). + function spliceOne(list, index) { + for (var i = index, k = i + 1, n = list.length; k < n; i += 1, k += 1) + list[i] = list[k]; + list.pop(); + } + + function arrayClone(arr, i) { + var copy = new Array(i); + while (i--) + copy[i] = arr[i]; + return copy; + } + + function unwrapListeners(arr) { + var ret = new Array(arr.length); + for (var i = 0; i < ret.length; ++i) { + ret[i] = arr[i].listener || arr[i]; + } + return ret; + } + + var global$1 = (typeof global !== "undefined" ? global : + typeof self !== "undefined" ? self : + typeof window !== "undefined" ? window : {}); + + var lookup = []; + var revLookup = []; + var Arr = typeof Uint8Array !== 'undefined' ? Uint8Array : Array; + var inited = false; + function init () { + inited = true; + var code = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'; + for (var i = 0, len = code.length; i < len; ++i) { + lookup[i] = code[i]; + revLookup[code.charCodeAt(i)] = i; + } + + revLookup['-'.charCodeAt(0)] = 62; + revLookup['_'.charCodeAt(0)] = 63; + } + + function toByteArray (b64) { + if (!inited) { + init(); + } + var i, j, l, tmp, placeHolders, arr; + var len = b64.length; + + if (len % 4 > 0) { + throw new Error('Invalid string. Length must be a multiple of 4') + } + + // the number of equal signs (place holders) + // if there are two placeholders, than the two characters before it + // represent one byte + // if there is only one, then the three characters before it represent 2 bytes + // this is just a cheap hack to not do indexOf twice + placeHolders = b64[len - 2] === '=' ? 2 : b64[len - 1] === '=' ? 1 : 0; + + // base64 is 4/3 + up to two characters of the original data + arr = new Arr(len * 3 / 4 - placeHolders); + + // if there are placeholders, only get up to the last complete 4 chars + l = placeHolders > 0 ? len - 4 : len; + + var L = 0; + + for (i = 0, j = 0; i < l; i += 4, j += 3) { + tmp = (revLookup[b64.charCodeAt(i)] << 18) | (revLookup[b64.charCodeAt(i + 1)] << 12) | (revLookup[b64.charCodeAt(i + 2)] << 6) | revLookup[b64.charCodeAt(i + 3)]; + arr[L++] = (tmp >> 16) & 0xFF; + arr[L++] = (tmp >> 8) & 0xFF; + arr[L++] = tmp & 0xFF; + } + + if (placeHolders === 2) { + tmp = (revLookup[b64.charCodeAt(i)] << 2) | (revLookup[b64.charCodeAt(i + 1)] >> 4); + arr[L++] = tmp & 0xFF; + } else if (placeHolders === 1) { + tmp = (revLookup[b64.charCodeAt(i)] << 10) | (revLookup[b64.charCodeAt(i + 1)] << 4) | (revLookup[b64.charCodeAt(i + 2)] >> 2); + arr[L++] = (tmp >> 8) & 0xFF; + arr[L++] = tmp & 0xFF; + } + + return arr + } + + function tripletToBase64 (num) { + return lookup[num >> 18 & 0x3F] + lookup[num >> 12 & 0x3F] + lookup[num >> 6 & 0x3F] + lookup[num & 0x3F] + } + + function encodeChunk (uint8, start, end) { + var tmp; + var output = []; + for (var i = start; i < end; i += 3) { + tmp = (uint8[i] << 16) + (uint8[i + 1] << 8) + (uint8[i + 2]); + output.push(tripletToBase64(tmp)); + } + return output.join('') + } + + function fromByteArray (uint8) { + if (!inited) { + init(); + } + var tmp; + var len = uint8.length; + var extraBytes = len % 3; // if we have 1 byte left, pad 2 bytes + var output = ''; + var parts = []; + var maxChunkLength = 16383; // must be multiple of 3 + + // go through the array every three bytes, we'll deal with trailing stuff later + for (var i = 0, len2 = len - extraBytes; i < len2; i += maxChunkLength) { + parts.push(encodeChunk(uint8, i, (i + maxChunkLength) > len2 ? len2 : (i + maxChunkLength))); + } + + // pad the end with zeros, but make sure to not forget the extra bytes + if (extraBytes === 1) { + tmp = uint8[len - 1]; + output += lookup[tmp >> 2]; + output += lookup[(tmp << 4) & 0x3F]; + output += '=='; + } else if (extraBytes === 2) { + tmp = (uint8[len - 2] << 8) + (uint8[len - 1]); + output += lookup[tmp >> 10]; + output += lookup[(tmp >> 4) & 0x3F]; + output += lookup[(tmp << 2) & 0x3F]; + output += '='; + } + + parts.push(output); + + return parts.join('') + } + + function read (buffer, offset, isLE, mLen, nBytes) { + var e, m; + var eLen = nBytes * 8 - mLen - 1; + var eMax = (1 << eLen) - 1; + var eBias = eMax >> 1; + var nBits = -7; + var i = isLE ? (nBytes - 1) : 0; + var d = isLE ? -1 : 1; + var s = buffer[offset + i]; + + i += d; + + e = s & ((1 << (-nBits)) - 1); + s >>= (-nBits); + nBits += eLen; + for (; nBits > 0; e = e * 256 + buffer[offset + i], i += d, nBits -= 8) {} + + m = e & ((1 << (-nBits)) - 1); + e >>= (-nBits); + nBits += mLen; + for (; nBits > 0; m = m * 256 + buffer[offset + i], i += d, nBits -= 8) {} + + if (e === 0) { + e = 1 - eBias; + } else if (e === eMax) { + return m ? NaN : ((s ? -1 : 1) * Infinity) + } else { + m = m + Math.pow(2, mLen); + e = e - eBias; + } + return (s ? -1 : 1) * m * Math.pow(2, e - mLen) + } + + function write (buffer, value, offset, isLE, mLen, nBytes) { + var e, m, c; + var eLen = nBytes * 8 - mLen - 1; + var eMax = (1 << eLen) - 1; + var eBias = eMax >> 1; + var rt = (mLen === 23 ? Math.pow(2, -24) - Math.pow(2, -77) : 0); + var i = isLE ? 0 : (nBytes - 1); + var d = isLE ? 1 : -1; + var s = value < 0 || (value === 0 && 1 / value < 0) ? 1 : 0; + + value = Math.abs(value); + + if (isNaN(value) || value === Infinity) { + m = isNaN(value) ? 1 : 0; + e = eMax; + } else { + e = Math.floor(Math.log(value) / Math.LN2); + if (value * (c = Math.pow(2, -e)) < 1) { + e--; + c *= 2; + } + if (e + eBias >= 1) { + value += rt / c; + } else { + value += rt * Math.pow(2, 1 - eBias); + } + if (value * c >= 2) { + e++; + c /= 2; + } + + if (e + eBias >= eMax) { + m = 0; + e = eMax; + } else if (e + eBias >= 1) { + m = (value * c - 1) * Math.pow(2, mLen); + e = e + eBias; + } else { + m = value * Math.pow(2, eBias - 1) * Math.pow(2, mLen); + e = 0; + } + } + + for (; mLen >= 8; buffer[offset + i] = m & 0xff, i += d, m /= 256, mLen -= 8) {} + + e = (e << mLen) | m; + eLen += mLen; + for (; eLen > 0; buffer[offset + i] = e & 0xff, i += d, e /= 256, eLen -= 8) {} + + buffer[offset + i - d] |= s * 128; + } + + var toString = {}.toString; + + var isArray$1 = Array.isArray || function (arr) { + return toString.call(arr) == '[object Array]'; + }; + + var INSPECT_MAX_BYTES = 50; + + /** + * If `Buffer.TYPED_ARRAY_SUPPORT`: + * === true Use Uint8Array implementation (fastest) + * === false Use Object implementation (most compatible, even IE6) + * + * Browsers that support typed arrays are IE 10+, Firefox 4+, Chrome 7+, Safari 5.1+, + * Opera 11.6+, iOS 4.2+. + * + * Due to various browser bugs, sometimes the Object implementation will be used even + * when the browser supports typed arrays. + * + * Note: + * + * - Firefox 4-29 lacks support for adding new properties to `Uint8Array` instances, + * See: https://bugzilla.mozilla.org/show_bug.cgi?id=695438. + * + * - Chrome 9-10 is missing the `TypedArray.prototype.subarray` function. + * + * - IE10 has a broken `TypedArray.prototype.subarray` function which returns arrays of + * incorrect length in some situations. + + * We detect these buggy browsers and set `Buffer.TYPED_ARRAY_SUPPORT` to `false` so they + * get the Object implementation, which is slower but behaves correctly. + */ + Buffer.TYPED_ARRAY_SUPPORT = global$1.TYPED_ARRAY_SUPPORT !== undefined + ? global$1.TYPED_ARRAY_SUPPORT + : true; + + function kMaxLength () { + return Buffer.TYPED_ARRAY_SUPPORT + ? 0x7fffffff + : 0x3fffffff + } + + function createBuffer (that, length) { + if (kMaxLength() < length) { + throw new RangeError('Invalid typed array length') + } + if (Buffer.TYPED_ARRAY_SUPPORT) { + // Return an augmented `Uint8Array` instance, for best performance + that = new Uint8Array(length); + that.__proto__ = Buffer.prototype; + } else { + // Fallback: Return an object instance of the Buffer class + if (that === null) { + that = new Buffer(length); + } + that.length = length; + } + + return that + } + + /** + * The Buffer constructor returns instances of `Uint8Array` that have their + * prototype changed to `Buffer.prototype`. Furthermore, `Buffer` is a subclass of + * `Uint8Array`, so the returned instances will have all the node `Buffer` methods + * and the `Uint8Array` methods. Square bracket notation works as expected -- it + * returns a single octet. + * + * The `Uint8Array` prototype remains unmodified. + */ + + function Buffer (arg, encodingOrOffset, length) { + if (!Buffer.TYPED_ARRAY_SUPPORT && !(this instanceof Buffer)) { + return new Buffer(arg, encodingOrOffset, length) + } + + // Common case. + if (typeof arg === 'number') { + if (typeof encodingOrOffset === 'string') { + throw new Error( + 'If encoding is specified then the first argument must be a string' + ) + } + return allocUnsafe(this, arg) + } + return from(this, arg, encodingOrOffset, length) + } + + Buffer.poolSize = 8192; // not used by this implementation + + // TODO: Legacy, not needed anymore. Remove in next major version. + Buffer._augment = function (arr) { + arr.__proto__ = Buffer.prototype; + return arr + }; + + function from (that, value, encodingOrOffset, length) { + if (typeof value === 'number') { + throw new TypeError('"value" argument must not be a number') + } + + if (typeof ArrayBuffer !== 'undefined' && value instanceof ArrayBuffer) { + return fromArrayBuffer(that, value, encodingOrOffset, length) + } + + if (typeof value === 'string') { + return fromString(that, value, encodingOrOffset) + } + + return fromObject(that, value) + } + + /** + * Functionally equivalent to Buffer(arg, encoding) but throws a TypeError + * if value is a number. + * Buffer.from(str[, encoding]) + * Buffer.from(array) + * Buffer.from(buffer) + * Buffer.from(arrayBuffer[, byteOffset[, length]]) + **/ + Buffer.from = function (value, encodingOrOffset, length) { + return from(null, value, encodingOrOffset, length) + }; + + if (Buffer.TYPED_ARRAY_SUPPORT) { + Buffer.prototype.__proto__ = Uint8Array.prototype; + Buffer.__proto__ = Uint8Array; + } + + function assertSize (size) { + if (typeof size !== 'number') { + throw new TypeError('"size" argument must be a number') + } else if (size < 0) { + throw new RangeError('"size" argument must not be negative') + } + } + + function alloc (that, size, fill, encoding) { + assertSize(size); + if (size <= 0) { + return createBuffer(that, size) + } + if (fill !== undefined) { + // Only pay attention to encoding if it's a string. This + // prevents accidentally sending in a number that would + // be interpretted as a start offset. + return typeof encoding === 'string' + ? createBuffer(that, size).fill(fill, encoding) + : createBuffer(that, size).fill(fill) + } + return createBuffer(that, size) + } + + /** + * Creates a new filled Buffer instance. + * alloc(size[, fill[, encoding]]) + **/ + Buffer.alloc = function (size, fill, encoding) { + return alloc(null, size, fill, encoding) + }; + + function allocUnsafe (that, size) { + assertSize(size); + that = createBuffer(that, size < 0 ? 0 : checked(size) | 0); + if (!Buffer.TYPED_ARRAY_SUPPORT) { + for (var i = 0; i < size; ++i) { + that[i] = 0; + } + } + return that + } + + /** + * Equivalent to Buffer(num), by default creates a non-zero-filled Buffer instance. + * */ + Buffer.allocUnsafe = function (size) { + return allocUnsafe(null, size) + }; + /** + * Equivalent to SlowBuffer(num), by default creates a non-zero-filled Buffer instance. + */ + Buffer.allocUnsafeSlow = function (size) { + return allocUnsafe(null, size) + }; + + function fromString (that, string, encoding) { + if (typeof encoding !== 'string' || encoding === '') { + encoding = 'utf8'; + } + + if (!Buffer.isEncoding(encoding)) { + throw new TypeError('"encoding" must be a valid string encoding') + } + + var length = byteLength(string, encoding) | 0; + that = createBuffer(that, length); + + var actual = that.write(string, encoding); + + if (actual !== length) { + // Writing a hex string, for example, that contains invalid characters will + // cause everything after the first invalid character to be ignored. (e.g. + // 'abxxcd' will be treated as 'ab') + that = that.slice(0, actual); + } + + return that + } + + function fromArrayLike (that, array) { + var length = array.length < 0 ? 0 : checked(array.length) | 0; + that = createBuffer(that, length); + for (var i = 0; i < length; i += 1) { + that[i] = array[i] & 255; + } + return that + } + + function fromArrayBuffer (that, array, byteOffset, length) { + array.byteLength; // this throws if `array` is not a valid ArrayBuffer + + if (byteOffset < 0 || array.byteLength < byteOffset) { + throw new RangeError('\'offset\' is out of bounds') + } + + if (array.byteLength < byteOffset + (length || 0)) { + throw new RangeError('\'length\' is out of bounds') + } + + if (byteOffset === undefined && length === undefined) { + array = new Uint8Array(array); + } else if (length === undefined) { + array = new Uint8Array(array, byteOffset); + } else { + array = new Uint8Array(array, byteOffset, length); + } + + if (Buffer.TYPED_ARRAY_SUPPORT) { + // Return an augmented `Uint8Array` instance, for best performance + that = array; + that.__proto__ = Buffer.prototype; + } else { + // Fallback: Return an object instance of the Buffer class + that = fromArrayLike(that, array); + } + return that + } + + function fromObject (that, obj) { + if (internalIsBuffer(obj)) { + var len = checked(obj.length) | 0; + that = createBuffer(that, len); + + if (that.length === 0) { + return that + } + + obj.copy(that, 0, 0, len); + return that + } + + if (obj) { + if ((typeof ArrayBuffer !== 'undefined' && + obj.buffer instanceof ArrayBuffer) || 'length' in obj) { + if (typeof obj.length !== 'number' || isnan(obj.length)) { + return createBuffer(that, 0) + } + return fromArrayLike(that, obj) + } + + if (obj.type === 'Buffer' && isArray$1(obj.data)) { + return fromArrayLike(that, obj.data) + } + } + + throw new TypeError('First argument must be a string, Buffer, ArrayBuffer, Array, or array-like object.') + } + + function checked (length) { + // Note: cannot use `length < kMaxLength()` here because that fails when + // length is NaN (which is otherwise coerced to zero.) + if (length >= kMaxLength()) { + throw new RangeError('Attempt to allocate Buffer larger than maximum ' + + 'size: 0x' + kMaxLength().toString(16) + ' bytes') + } + return length | 0 + } + Buffer.isBuffer = isBuffer; + function internalIsBuffer (b) { + return !!(b != null && b._isBuffer) + } + + Buffer.compare = function compare (a, b) { + if (!internalIsBuffer(a) || !internalIsBuffer(b)) { + throw new TypeError('Arguments must be Buffers') + } + + if (a === b) return 0 + + var x = a.length; + var y = b.length; + + for (var i = 0, len = Math.min(x, y); i < len; ++i) { + if (a[i] !== b[i]) { + x = a[i]; + y = b[i]; + break + } + } + + if (x < y) return -1 + if (y < x) return 1 + return 0 + }; + + Buffer.isEncoding = function isEncoding (encoding) { + switch (String(encoding).toLowerCase()) { + case 'hex': + case 'utf8': + case 'utf-8': + case 'ascii': + case 'latin1': + case 'binary': + case 'base64': + case 'ucs2': + case 'ucs-2': + case 'utf16le': + case 'utf-16le': + return true + default: + return false + } + }; + + Buffer.concat = function concat (list, length) { + if (!isArray$1(list)) { + throw new TypeError('"list" argument must be an Array of Buffers') + } + + if (list.length === 0) { + return Buffer.alloc(0) + } + + var i; + if (length === undefined) { + length = 0; + for (i = 0; i < list.length; ++i) { + length += list[i].length; + } + } + + var buffer = Buffer.allocUnsafe(length); + var pos = 0; + for (i = 0; i < list.length; ++i) { + var buf = list[i]; + if (!internalIsBuffer(buf)) { + throw new TypeError('"list" argument must be an Array of Buffers') + } + buf.copy(buffer, pos); + pos += buf.length; + } + return buffer + }; + + function byteLength (string, encoding) { + if (internalIsBuffer(string)) { + return string.length + } + if (typeof ArrayBuffer !== 'undefined' && typeof ArrayBuffer.isView === 'function' && + (ArrayBuffer.isView(string) || string instanceof ArrayBuffer)) { + return string.byteLength + } + if (typeof string !== 'string') { + string = '' + string; + } + + var len = string.length; + if (len === 0) return 0 + + // Use a for loop to avoid recursion + var loweredCase = false; + for (;;) { + switch (encoding) { + case 'ascii': + case 'latin1': + case 'binary': + return len + case 'utf8': + case 'utf-8': + case undefined: + return utf8ToBytes(string).length + case 'ucs2': + case 'ucs-2': + case 'utf16le': + case 'utf-16le': + return len * 2 + case 'hex': + return len >>> 1 + case 'base64': + return base64ToBytes(string).length + default: + if (loweredCase) return utf8ToBytes(string).length // assume utf8 + encoding = ('' + encoding).toLowerCase(); + loweredCase = true; + } + } + } + Buffer.byteLength = byteLength; + + function slowToString (encoding, start, end) { + var loweredCase = false; + + // No need to verify that "this.length <= MAX_UINT32" since it's a read-only + // property of a typed array. + + // This behaves neither like String nor Uint8Array in that we set start/end + // to their upper/lower bounds if the value passed is out of range. + // undefined is handled specially as per ECMA-262 6th Edition, + // Section 13.3.3.7 Runtime Semantics: KeyedBindingInitialization. + if (start === undefined || start < 0) { + start = 0; + } + // Return early if start > this.length. Done here to prevent potential uint32 + // coercion fail below. + if (start > this.length) { + return '' + } + + if (end === undefined || end > this.length) { + end = this.length; + } + + if (end <= 0) { + return '' + } + + // Force coersion to uint32. This will also coerce falsey/NaN values to 0. + end >>>= 0; + start >>>= 0; + + if (end <= start) { + return '' + } + + if (!encoding) encoding = 'utf8'; + + while (true) { + switch (encoding) { + case 'hex': + return hexSlice(this, start, end) + + case 'utf8': + case 'utf-8': + return utf8Slice(this, start, end) + + case 'ascii': + return asciiSlice(this, start, end) + + case 'latin1': + case 'binary': + return latin1Slice(this, start, end) + + case 'base64': + return base64Slice(this, start, end) + + case 'ucs2': + case 'ucs-2': + case 'utf16le': + case 'utf-16le': + return utf16leSlice(this, start, end) + + default: + if (loweredCase) throw new TypeError('Unknown encoding: ' + encoding) + encoding = (encoding + '').toLowerCase(); + loweredCase = true; + } + } + } + + // The property is used by `Buffer.isBuffer` and `is-buffer` (in Safari 5-7) to detect + // Buffer instances. + Buffer.prototype._isBuffer = true; + + function swap (b, n, m) { + var i = b[n]; + b[n] = b[m]; + b[m] = i; + } + + Buffer.prototype.swap16 = function swap16 () { + var len = this.length; + if (len % 2 !== 0) { + throw new RangeError('Buffer size must be a multiple of 16-bits') + } + for (var i = 0; i < len; i += 2) { + swap(this, i, i + 1); + } + return this + }; + + Buffer.prototype.swap32 = function swap32 () { + var len = this.length; + if (len % 4 !== 0) { + throw new RangeError('Buffer size must be a multiple of 32-bits') + } + for (var i = 0; i < len; i += 4) { + swap(this, i, i + 3); + swap(this, i + 1, i + 2); + } + return this + }; + + Buffer.prototype.swap64 = function swap64 () { + var len = this.length; + if (len % 8 !== 0) { + throw new RangeError('Buffer size must be a multiple of 64-bits') + } + for (var i = 0; i < len; i += 8) { + swap(this, i, i + 7); + swap(this, i + 1, i + 6); + swap(this, i + 2, i + 5); + swap(this, i + 3, i + 4); + } + return this + }; + + Buffer.prototype.toString = function toString () { + var length = this.length | 0; + if (length === 0) return '' + if (arguments.length === 0) return utf8Slice(this, 0, length) + return slowToString.apply(this, arguments) + }; + + Buffer.prototype.equals = function equals (b) { + if (!internalIsBuffer(b)) throw new TypeError('Argument must be a Buffer') + if (this === b) return true + return Buffer.compare(this, b) === 0 + }; + + Buffer.prototype.inspect = function inspect () { + var str = ''; + var max = INSPECT_MAX_BYTES; + if (this.length > 0) { + str = this.toString('hex', 0, max).match(/.{2}/g).join(' '); + if (this.length > max) str += ' ... '; + } + return '' + }; + + Buffer.prototype.compare = function compare (target, start, end, thisStart, thisEnd) { + if (!internalIsBuffer(target)) { + throw new TypeError('Argument must be a Buffer') + } + + if (start === undefined) { + start = 0; + } + if (end === undefined) { + end = target ? target.length : 0; + } + if (thisStart === undefined) { + thisStart = 0; + } + if (thisEnd === undefined) { + thisEnd = this.length; + } + + if (start < 0 || end > target.length || thisStart < 0 || thisEnd > this.length) { + throw new RangeError('out of range index') + } + + if (thisStart >= thisEnd && start >= end) { + return 0 + } + if (thisStart >= thisEnd) { + return -1 + } + if (start >= end) { + return 1 + } + + start >>>= 0; + end >>>= 0; + thisStart >>>= 0; + thisEnd >>>= 0; + + if (this === target) return 0 + + var x = thisEnd - thisStart; + var y = end - start; + var len = Math.min(x, y); + + var thisCopy = this.slice(thisStart, thisEnd); + var targetCopy = target.slice(start, end); + + for (var i = 0; i < len; ++i) { + if (thisCopy[i] !== targetCopy[i]) { + x = thisCopy[i]; + y = targetCopy[i]; + break + } + } + + if (x < y) return -1 + if (y < x) return 1 + return 0 + }; + + // Finds either the first index of `val` in `buffer` at offset >= `byteOffset`, + // OR the last index of `val` in `buffer` at offset <= `byteOffset`. + // + // Arguments: + // - buffer - a Buffer to search + // - val - a string, Buffer, or number + // - byteOffset - an index into `buffer`; will be clamped to an int32 + // - encoding - an optional encoding, relevant is val is a string + // - dir - true for indexOf, false for lastIndexOf + function bidirectionalIndexOf (buffer, val, byteOffset, encoding, dir) { + // Empty buffer means no match + if (buffer.length === 0) return -1 + + // Normalize byteOffset + if (typeof byteOffset === 'string') { + encoding = byteOffset; + byteOffset = 0; + } else if (byteOffset > 0x7fffffff) { + byteOffset = 0x7fffffff; + } else if (byteOffset < -0x80000000) { + byteOffset = -0x80000000; + } + byteOffset = +byteOffset; // Coerce to Number. + if (isNaN(byteOffset)) { + // byteOffset: it it's undefined, null, NaN, "foo", etc, search whole buffer + byteOffset = dir ? 0 : (buffer.length - 1); + } + + // Normalize byteOffset: negative offsets start from the end of the buffer + if (byteOffset < 0) byteOffset = buffer.length + byteOffset; + if (byteOffset >= buffer.length) { + if (dir) return -1 + else byteOffset = buffer.length - 1; + } else if (byteOffset < 0) { + if (dir) byteOffset = 0; + else return -1 + } + + // Normalize val + if (typeof val === 'string') { + val = Buffer.from(val, encoding); + } + + // Finally, search either indexOf (if dir is true) or lastIndexOf + if (internalIsBuffer(val)) { + // Special case: looking for empty string/buffer always fails + if (val.length === 0) { + return -1 + } + return arrayIndexOf(buffer, val, byteOffset, encoding, dir) + } else if (typeof val === 'number') { + val = val & 0xFF; // Search for a byte value [0-255] + if (Buffer.TYPED_ARRAY_SUPPORT && + typeof Uint8Array.prototype.indexOf === 'function') { + if (dir) { + return Uint8Array.prototype.indexOf.call(buffer, val, byteOffset) + } else { + return Uint8Array.prototype.lastIndexOf.call(buffer, val, byteOffset) + } + } + return arrayIndexOf(buffer, [ val ], byteOffset, encoding, dir) + } + + throw new TypeError('val must be string, number or Buffer') + } + + function arrayIndexOf (arr, val, byteOffset, encoding, dir) { + var indexSize = 1; + var arrLength = arr.length; + var valLength = val.length; + + if (encoding !== undefined) { + encoding = String(encoding).toLowerCase(); + if (encoding === 'ucs2' || encoding === 'ucs-2' || + encoding === 'utf16le' || encoding === 'utf-16le') { + if (arr.length < 2 || val.length < 2) { + return -1 + } + indexSize = 2; + arrLength /= 2; + valLength /= 2; + byteOffset /= 2; + } + } + + function read (buf, i) { + if (indexSize === 1) { + return buf[i] + } else { + return buf.readUInt16BE(i * indexSize) + } + } + + var i; + if (dir) { + var foundIndex = -1; + for (i = byteOffset; i < arrLength; i++) { + if (read(arr, i) === read(val, foundIndex === -1 ? 0 : i - foundIndex)) { + if (foundIndex === -1) foundIndex = i; + if (i - foundIndex + 1 === valLength) return foundIndex * indexSize + } else { + if (foundIndex !== -1) i -= i - foundIndex; + foundIndex = -1; + } + } + } else { + if (byteOffset + valLength > arrLength) byteOffset = arrLength - valLength; + for (i = byteOffset; i >= 0; i--) { + var found = true; + for (var j = 0; j < valLength; j++) { + if (read(arr, i + j) !== read(val, j)) { + found = false; + break + } + } + if (found) return i + } + } + + return -1 + } + + Buffer.prototype.includes = function includes (val, byteOffset, encoding) { + return this.indexOf(val, byteOffset, encoding) !== -1 + }; + + Buffer.prototype.indexOf = function indexOf (val, byteOffset, encoding) { + return bidirectionalIndexOf(this, val, byteOffset, encoding, true) + }; + + Buffer.prototype.lastIndexOf = function lastIndexOf (val, byteOffset, encoding) { + return bidirectionalIndexOf(this, val, byteOffset, encoding, false) + }; + + function hexWrite (buf, string, offset, length) { + offset = Number(offset) || 0; + var remaining = buf.length - offset; + if (!length) { + length = remaining; + } else { + length = Number(length); + if (length > remaining) { + length = remaining; + } + } + + // must be an even number of digits + var strLen = string.length; + if (strLen % 2 !== 0) throw new TypeError('Invalid hex string') + + if (length > strLen / 2) { + length = strLen / 2; + } + for (var i = 0; i < length; ++i) { + var parsed = parseInt(string.substr(i * 2, 2), 16); + if (isNaN(parsed)) return i + buf[offset + i] = parsed; + } + return i + } + + function utf8Write (buf, string, offset, length) { + return blitBuffer(utf8ToBytes(string, buf.length - offset), buf, offset, length) + } + + function asciiWrite (buf, string, offset, length) { + return blitBuffer(asciiToBytes(string), buf, offset, length) + } + + function latin1Write (buf, string, offset, length) { + return asciiWrite(buf, string, offset, length) + } + + function base64Write (buf, string, offset, length) { + return blitBuffer(base64ToBytes(string), buf, offset, length) + } + + function ucs2Write (buf, string, offset, length) { + return blitBuffer(utf16leToBytes(string, buf.length - offset), buf, offset, length) + } + + Buffer.prototype.write = function write (string, offset, length, encoding) { + // Buffer#write(string) + if (offset === undefined) { + encoding = 'utf8'; + length = this.length; + offset = 0; + // Buffer#write(string, encoding) + } else if (length === undefined && typeof offset === 'string') { + encoding = offset; + length = this.length; + offset = 0; + // Buffer#write(string, offset[, length][, encoding]) + } else if (isFinite(offset)) { + offset = offset | 0; + if (isFinite(length)) { + length = length | 0; + if (encoding === undefined) encoding = 'utf8'; + } else { + encoding = length; + length = undefined; + } + // legacy write(string, encoding, offset, length) - remove in v0.13 + } else { + throw new Error( + 'Buffer.write(string, encoding, offset[, length]) is no longer supported' + ) + } + + var remaining = this.length - offset; + if (length === undefined || length > remaining) length = remaining; + + if ((string.length > 0 && (length < 0 || offset < 0)) || offset > this.length) { + throw new RangeError('Attempt to write outside buffer bounds') + } + + if (!encoding) encoding = 'utf8'; + + var loweredCase = false; + for (;;) { + switch (encoding) { + case 'hex': + return hexWrite(this, string, offset, length) + + case 'utf8': + case 'utf-8': + return utf8Write(this, string, offset, length) + + case 'ascii': + return asciiWrite(this, string, offset, length) + + case 'latin1': + case 'binary': + return latin1Write(this, string, offset, length) + + case 'base64': + // Warning: maxLength not taken into account in base64Write + return base64Write(this, string, offset, length) + + case 'ucs2': + case 'ucs-2': + case 'utf16le': + case 'utf-16le': + return ucs2Write(this, string, offset, length) + + default: + if (loweredCase) throw new TypeError('Unknown encoding: ' + encoding) + encoding = ('' + encoding).toLowerCase(); + loweredCase = true; + } + } + }; + + Buffer.prototype.toJSON = function toJSON () { + return { + type: 'Buffer', + data: Array.prototype.slice.call(this._arr || this, 0) + } + }; + + function base64Slice (buf, start, end) { + if (start === 0 && end === buf.length) { + return fromByteArray(buf) + } else { + return fromByteArray(buf.slice(start, end)) + } + } + + function utf8Slice (buf, start, end) { + end = Math.min(buf.length, end); + var res = []; + + var i = start; + while (i < end) { + var firstByte = buf[i]; + var codePoint = null; + var bytesPerSequence = (firstByte > 0xEF) ? 4 + : (firstByte > 0xDF) ? 3 + : (firstByte > 0xBF) ? 2 + : 1; + + if (i + bytesPerSequence <= end) { + var secondByte, thirdByte, fourthByte, tempCodePoint; + + switch (bytesPerSequence) { + case 1: + if (firstByte < 0x80) { + codePoint = firstByte; + } + break + case 2: + secondByte = buf[i + 1]; + if ((secondByte & 0xC0) === 0x80) { + tempCodePoint = (firstByte & 0x1F) << 0x6 | (secondByte & 0x3F); + if (tempCodePoint > 0x7F) { + codePoint = tempCodePoint; + } + } + break + case 3: + secondByte = buf[i + 1]; + thirdByte = buf[i + 2]; + if ((secondByte & 0xC0) === 0x80 && (thirdByte & 0xC0) === 0x80) { + tempCodePoint = (firstByte & 0xF) << 0xC | (secondByte & 0x3F) << 0x6 | (thirdByte & 0x3F); + if (tempCodePoint > 0x7FF && (tempCodePoint < 0xD800 || tempCodePoint > 0xDFFF)) { + codePoint = tempCodePoint; + } + } + break + case 4: + secondByte = buf[i + 1]; + thirdByte = buf[i + 2]; + fourthByte = buf[i + 3]; + if ((secondByte & 0xC0) === 0x80 && (thirdByte & 0xC0) === 0x80 && (fourthByte & 0xC0) === 0x80) { + tempCodePoint = (firstByte & 0xF) << 0x12 | (secondByte & 0x3F) << 0xC | (thirdByte & 0x3F) << 0x6 | (fourthByte & 0x3F); + if (tempCodePoint > 0xFFFF && tempCodePoint < 0x110000) { + codePoint = tempCodePoint; + } + } + } + } + + if (codePoint === null) { + // we did not generate a valid codePoint so insert a + // replacement char (U+FFFD) and advance only 1 byte + codePoint = 0xFFFD; + bytesPerSequence = 1; + } else if (codePoint > 0xFFFF) { + // encode to utf16 (surrogate pair dance) + codePoint -= 0x10000; + res.push(codePoint >>> 10 & 0x3FF | 0xD800); + codePoint = 0xDC00 | codePoint & 0x3FF; + } + + res.push(codePoint); + i += bytesPerSequence; + } + + return decodeCodePointsArray(res) + } + + // Based on http://stackoverflow.com/a/22747272/680742, the browser with + // the lowest limit is Chrome, with 0x10000 args. + // We go 1 magnitude less, for safety + var MAX_ARGUMENTS_LENGTH = 0x1000; + + function decodeCodePointsArray (codePoints) { + var len = codePoints.length; + if (len <= MAX_ARGUMENTS_LENGTH) { + return String.fromCharCode.apply(String, codePoints) // avoid extra slice() + } + + // Decode in chunks to avoid "call stack size exceeded". + var res = ''; + var i = 0; + while (i < len) { + res += String.fromCharCode.apply( + String, + codePoints.slice(i, i += MAX_ARGUMENTS_LENGTH) + ); + } + return res + } + + function asciiSlice (buf, start, end) { + var ret = ''; + end = Math.min(buf.length, end); + + for (var i = start; i < end; ++i) { + ret += String.fromCharCode(buf[i] & 0x7F); + } + return ret + } + + function latin1Slice (buf, start, end) { + var ret = ''; + end = Math.min(buf.length, end); + + for (var i = start; i < end; ++i) { + ret += String.fromCharCode(buf[i]); + } + return ret + } + + function hexSlice (buf, start, end) { + var len = buf.length; + + if (!start || start < 0) start = 0; + if (!end || end < 0 || end > len) end = len; + + var out = ''; + for (var i = start; i < end; ++i) { + out += toHex(buf[i]); + } + return out + } + + function utf16leSlice (buf, start, end) { + var bytes = buf.slice(start, end); + var res = ''; + for (var i = 0; i < bytes.length; i += 2) { + res += String.fromCharCode(bytes[i] + bytes[i + 1] * 256); + } + return res + } + + Buffer.prototype.slice = function slice (start, end) { + var len = this.length; + start = ~~start; + end = end === undefined ? len : ~~end; + + if (start < 0) { + start += len; + if (start < 0) start = 0; + } else if (start > len) { + start = len; + } + + if (end < 0) { + end += len; + if (end < 0) end = 0; + } else if (end > len) { + end = len; + } + + if (end < start) end = start; + + var newBuf; + if (Buffer.TYPED_ARRAY_SUPPORT) { + newBuf = this.subarray(start, end); + newBuf.__proto__ = Buffer.prototype; + } else { + var sliceLen = end - start; + newBuf = new Buffer(sliceLen, undefined); + for (var i = 0; i < sliceLen; ++i) { + newBuf[i] = this[i + start]; + } + } + + return newBuf + }; + + /* + * Need to make sure that buffer isn't trying to write out of bounds. + */ + function checkOffset (offset, ext, length) { + if ((offset % 1) !== 0 || offset < 0) throw new RangeError('offset is not uint') + if (offset + ext > length) throw new RangeError('Trying to access beyond buffer length') + } + + Buffer.prototype.readUIntLE = function readUIntLE (offset, byteLength, noAssert) { + offset = offset | 0; + byteLength = byteLength | 0; + if (!noAssert) checkOffset(offset, byteLength, this.length); + + var val = this[offset]; + var mul = 1; + var i = 0; + while (++i < byteLength && (mul *= 0x100)) { + val += this[offset + i] * mul; + } + + return val + }; + + Buffer.prototype.readUIntBE = function readUIntBE (offset, byteLength, noAssert) { + offset = offset | 0; + byteLength = byteLength | 0; + if (!noAssert) { + checkOffset(offset, byteLength, this.length); + } + + var val = this[offset + --byteLength]; + var mul = 1; + while (byteLength > 0 && (mul *= 0x100)) { + val += this[offset + --byteLength] * mul; + } + + return val + }; + + Buffer.prototype.readUInt8 = function readUInt8 (offset, noAssert) { + if (!noAssert) checkOffset(offset, 1, this.length); + return this[offset] + }; + + Buffer.prototype.readUInt16LE = function readUInt16LE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 2, this.length); + return this[offset] | (this[offset + 1] << 8) + }; + + Buffer.prototype.readUInt16BE = function readUInt16BE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 2, this.length); + return (this[offset] << 8) | this[offset + 1] + }; + + Buffer.prototype.readUInt32LE = function readUInt32LE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 4, this.length); + + return ((this[offset]) | + (this[offset + 1] << 8) | + (this[offset + 2] << 16)) + + (this[offset + 3] * 0x1000000) + }; + + Buffer.prototype.readUInt32BE = function readUInt32BE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 4, this.length); + + return (this[offset] * 0x1000000) + + ((this[offset + 1] << 16) | + (this[offset + 2] << 8) | + this[offset + 3]) + }; + + Buffer.prototype.readIntLE = function readIntLE (offset, byteLength, noAssert) { + offset = offset | 0; + byteLength = byteLength | 0; + if (!noAssert) checkOffset(offset, byteLength, this.length); + + var val = this[offset]; + var mul = 1; + var i = 0; + while (++i < byteLength && (mul *= 0x100)) { + val += this[offset + i] * mul; + } + mul *= 0x80; + + if (val >= mul) val -= Math.pow(2, 8 * byteLength); + + return val + }; + + Buffer.prototype.readIntBE = function readIntBE (offset, byteLength, noAssert) { + offset = offset | 0; + byteLength = byteLength | 0; + if (!noAssert) checkOffset(offset, byteLength, this.length); + + var i = byteLength; + var mul = 1; + var val = this[offset + --i]; + while (i > 0 && (mul *= 0x100)) { + val += this[offset + --i] * mul; + } + mul *= 0x80; + + if (val >= mul) val -= Math.pow(2, 8 * byteLength); + + return val + }; + + Buffer.prototype.readInt8 = function readInt8 (offset, noAssert) { + if (!noAssert) checkOffset(offset, 1, this.length); + if (!(this[offset] & 0x80)) return (this[offset]) + return ((0xff - this[offset] + 1) * -1) + }; + + Buffer.prototype.readInt16LE = function readInt16LE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 2, this.length); + var val = this[offset] | (this[offset + 1] << 8); + return (val & 0x8000) ? val | 0xFFFF0000 : val + }; + + Buffer.prototype.readInt16BE = function readInt16BE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 2, this.length); + var val = this[offset + 1] | (this[offset] << 8); + return (val & 0x8000) ? val | 0xFFFF0000 : val + }; + + Buffer.prototype.readInt32LE = function readInt32LE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 4, this.length); + + return (this[offset]) | + (this[offset + 1] << 8) | + (this[offset + 2] << 16) | + (this[offset + 3] << 24) + }; + + Buffer.prototype.readInt32BE = function readInt32BE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 4, this.length); + + return (this[offset] << 24) | + (this[offset + 1] << 16) | + (this[offset + 2] << 8) | + (this[offset + 3]) + }; + + Buffer.prototype.readFloatLE = function readFloatLE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 4, this.length); + return read(this, offset, true, 23, 4) + }; + + Buffer.prototype.readFloatBE = function readFloatBE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 4, this.length); + return read(this, offset, false, 23, 4) + }; + + Buffer.prototype.readDoubleLE = function readDoubleLE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 8, this.length); + return read(this, offset, true, 52, 8) + }; + + Buffer.prototype.readDoubleBE = function readDoubleBE (offset, noAssert) { + if (!noAssert) checkOffset(offset, 8, this.length); + return read(this, offset, false, 52, 8) + }; + + function checkInt (buf, value, offset, ext, max, min) { + if (!internalIsBuffer(buf)) throw new TypeError('"buffer" argument must be a Buffer instance') + if (value > max || value < min) throw new RangeError('"value" argument is out of bounds') + if (offset + ext > buf.length) throw new RangeError('Index out of range') + } + + Buffer.prototype.writeUIntLE = function writeUIntLE (value, offset, byteLength, noAssert) { + value = +value; + offset = offset | 0; + byteLength = byteLength | 0; + if (!noAssert) { + var maxBytes = Math.pow(2, 8 * byteLength) - 1; + checkInt(this, value, offset, byteLength, maxBytes, 0); + } + + var mul = 1; + var i = 0; + this[offset] = value & 0xFF; + while (++i < byteLength && (mul *= 0x100)) { + this[offset + i] = (value / mul) & 0xFF; + } + + return offset + byteLength + }; + + Buffer.prototype.writeUIntBE = function writeUIntBE (value, offset, byteLength, noAssert) { + value = +value; + offset = offset | 0; + byteLength = byteLength | 0; + if (!noAssert) { + var maxBytes = Math.pow(2, 8 * byteLength) - 1; + checkInt(this, value, offset, byteLength, maxBytes, 0); + } + + var i = byteLength - 1; + var mul = 1; + this[offset + i] = value & 0xFF; + while (--i >= 0 && (mul *= 0x100)) { + this[offset + i] = (value / mul) & 0xFF; + } + + return offset + byteLength + }; + + Buffer.prototype.writeUInt8 = function writeUInt8 (value, offset, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) checkInt(this, value, offset, 1, 0xff, 0); + if (!Buffer.TYPED_ARRAY_SUPPORT) value = Math.floor(value); + this[offset] = (value & 0xff); + return offset + 1 + }; + + function objectWriteUInt16 (buf, value, offset, littleEndian) { + if (value < 0) value = 0xffff + value + 1; + for (var i = 0, j = Math.min(buf.length - offset, 2); i < j; ++i) { + buf[offset + i] = (value & (0xff << (8 * (littleEndian ? i : 1 - i)))) >>> + (littleEndian ? i : 1 - i) * 8; + } + } + + Buffer.prototype.writeUInt16LE = function writeUInt16LE (value, offset, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) checkInt(this, value, offset, 2, 0xffff, 0); + if (Buffer.TYPED_ARRAY_SUPPORT) { + this[offset] = (value & 0xff); + this[offset + 1] = (value >>> 8); + } else { + objectWriteUInt16(this, value, offset, true); + } + return offset + 2 + }; + + Buffer.prototype.writeUInt16BE = function writeUInt16BE (value, offset, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) checkInt(this, value, offset, 2, 0xffff, 0); + if (Buffer.TYPED_ARRAY_SUPPORT) { + this[offset] = (value >>> 8); + this[offset + 1] = (value & 0xff); + } else { + objectWriteUInt16(this, value, offset, false); + } + return offset + 2 + }; + + function objectWriteUInt32 (buf, value, offset, littleEndian) { + if (value < 0) value = 0xffffffff + value + 1; + for (var i = 0, j = Math.min(buf.length - offset, 4); i < j; ++i) { + buf[offset + i] = (value >>> (littleEndian ? i : 3 - i) * 8) & 0xff; + } + } + + Buffer.prototype.writeUInt32LE = function writeUInt32LE (value, offset, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) checkInt(this, value, offset, 4, 0xffffffff, 0); + if (Buffer.TYPED_ARRAY_SUPPORT) { + this[offset + 3] = (value >>> 24); + this[offset + 2] = (value >>> 16); + this[offset + 1] = (value >>> 8); + this[offset] = (value & 0xff); + } else { + objectWriteUInt32(this, value, offset, true); + } + return offset + 4 + }; + + Buffer.prototype.writeUInt32BE = function writeUInt32BE (value, offset, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) checkInt(this, value, offset, 4, 0xffffffff, 0); + if (Buffer.TYPED_ARRAY_SUPPORT) { + this[offset] = (value >>> 24); + this[offset + 1] = (value >>> 16); + this[offset + 2] = (value >>> 8); + this[offset + 3] = (value & 0xff); + } else { + objectWriteUInt32(this, value, offset, false); + } + return offset + 4 + }; + + Buffer.prototype.writeIntLE = function writeIntLE (value, offset, byteLength, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) { + var limit = Math.pow(2, 8 * byteLength - 1); + + checkInt(this, value, offset, byteLength, limit - 1, -limit); + } + + var i = 0; + var mul = 1; + var sub = 0; + this[offset] = value & 0xFF; + while (++i < byteLength && (mul *= 0x100)) { + if (value < 0 && sub === 0 && this[offset + i - 1] !== 0) { + sub = 1; + } + this[offset + i] = ((value / mul) >> 0) - sub & 0xFF; + } + + return offset + byteLength + }; + + Buffer.prototype.writeIntBE = function writeIntBE (value, offset, byteLength, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) { + var limit = Math.pow(2, 8 * byteLength - 1); + + checkInt(this, value, offset, byteLength, limit - 1, -limit); + } + + var i = byteLength - 1; + var mul = 1; + var sub = 0; + this[offset + i] = value & 0xFF; + while (--i >= 0 && (mul *= 0x100)) { + if (value < 0 && sub === 0 && this[offset + i + 1] !== 0) { + sub = 1; + } + this[offset + i] = ((value / mul) >> 0) - sub & 0xFF; + } + + return offset + byteLength + }; + + Buffer.prototype.writeInt8 = function writeInt8 (value, offset, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) checkInt(this, value, offset, 1, 0x7f, -0x80); + if (!Buffer.TYPED_ARRAY_SUPPORT) value = Math.floor(value); + if (value < 0) value = 0xff + value + 1; + this[offset] = (value & 0xff); + return offset + 1 + }; + + Buffer.prototype.writeInt16LE = function writeInt16LE (value, offset, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) checkInt(this, value, offset, 2, 0x7fff, -0x8000); + if (Buffer.TYPED_ARRAY_SUPPORT) { + this[offset] = (value & 0xff); + this[offset + 1] = (value >>> 8); + } else { + objectWriteUInt16(this, value, offset, true); + } + return offset + 2 + }; + + Buffer.prototype.writeInt16BE = function writeInt16BE (value, offset, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) checkInt(this, value, offset, 2, 0x7fff, -0x8000); + if (Buffer.TYPED_ARRAY_SUPPORT) { + this[offset] = (value >>> 8); + this[offset + 1] = (value & 0xff); + } else { + objectWriteUInt16(this, value, offset, false); + } + return offset + 2 + }; + + Buffer.prototype.writeInt32LE = function writeInt32LE (value, offset, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) checkInt(this, value, offset, 4, 0x7fffffff, -0x80000000); + if (Buffer.TYPED_ARRAY_SUPPORT) { + this[offset] = (value & 0xff); + this[offset + 1] = (value >>> 8); + this[offset + 2] = (value >>> 16); + this[offset + 3] = (value >>> 24); + } else { + objectWriteUInt32(this, value, offset, true); + } + return offset + 4 + }; + + Buffer.prototype.writeInt32BE = function writeInt32BE (value, offset, noAssert) { + value = +value; + offset = offset | 0; + if (!noAssert) checkInt(this, value, offset, 4, 0x7fffffff, -0x80000000); + if (value < 0) value = 0xffffffff + value + 1; + if (Buffer.TYPED_ARRAY_SUPPORT) { + this[offset] = (value >>> 24); + this[offset + 1] = (value >>> 16); + this[offset + 2] = (value >>> 8); + this[offset + 3] = (value & 0xff); + } else { + objectWriteUInt32(this, value, offset, false); + } + return offset + 4 + }; + + function checkIEEE754 (buf, value, offset, ext, max, min) { + if (offset + ext > buf.length) throw new RangeError('Index out of range') + if (offset < 0) throw new RangeError('Index out of range') + } + + function writeFloat (buf, value, offset, littleEndian, noAssert) { + if (!noAssert) { + checkIEEE754(buf, value, offset, 4); + } + write(buf, value, offset, littleEndian, 23, 4); + return offset + 4 + } + + Buffer.prototype.writeFloatLE = function writeFloatLE (value, offset, noAssert) { + return writeFloat(this, value, offset, true, noAssert) + }; + + Buffer.prototype.writeFloatBE = function writeFloatBE (value, offset, noAssert) { + return writeFloat(this, value, offset, false, noAssert) + }; + + function writeDouble (buf, value, offset, littleEndian, noAssert) { + if (!noAssert) { + checkIEEE754(buf, value, offset, 8); + } + write(buf, value, offset, littleEndian, 52, 8); + return offset + 8 + } + + Buffer.prototype.writeDoubleLE = function writeDoubleLE (value, offset, noAssert) { + return writeDouble(this, value, offset, true, noAssert) + }; + + Buffer.prototype.writeDoubleBE = function writeDoubleBE (value, offset, noAssert) { + return writeDouble(this, value, offset, false, noAssert) + }; + + // copy(targetBuffer, targetStart=0, sourceStart=0, sourceEnd=buffer.length) + Buffer.prototype.copy = function copy (target, targetStart, start, end) { + if (!start) start = 0; + if (!end && end !== 0) end = this.length; + if (targetStart >= target.length) targetStart = target.length; + if (!targetStart) targetStart = 0; + if (end > 0 && end < start) end = start; + + // Copy 0 bytes; we're done + if (end === start) return 0 + if (target.length === 0 || this.length === 0) return 0 + + // Fatal error conditions + if (targetStart < 0) { + throw new RangeError('targetStart out of bounds') + } + if (start < 0 || start >= this.length) throw new RangeError('sourceStart out of bounds') + if (end < 0) throw new RangeError('sourceEnd out of bounds') + + // Are we oob? + if (end > this.length) end = this.length; + if (target.length - targetStart < end - start) { + end = target.length - targetStart + start; + } + + var len = end - start; + var i; + + if (this === target && start < targetStart && targetStart < end) { + // descending copy from end + for (i = len - 1; i >= 0; --i) { + target[i + targetStart] = this[i + start]; + } + } else if (len < 1000 || !Buffer.TYPED_ARRAY_SUPPORT) { + // ascending copy from start + for (i = 0; i < len; ++i) { + target[i + targetStart] = this[i + start]; + } + } else { + Uint8Array.prototype.set.call( + target, + this.subarray(start, start + len), + targetStart + ); + } + + return len + }; + + // Usage: + // buffer.fill(number[, offset[, end]]) + // buffer.fill(buffer[, offset[, end]]) + // buffer.fill(string[, offset[, end]][, encoding]) + Buffer.prototype.fill = function fill (val, start, end, encoding) { + // Handle string cases: + if (typeof val === 'string') { + if (typeof start === 'string') { + encoding = start; + start = 0; + end = this.length; + } else if (typeof end === 'string') { + encoding = end; + end = this.length; + } + if (val.length === 1) { + var code = val.charCodeAt(0); + if (code < 256) { + val = code; + } + } + if (encoding !== undefined && typeof encoding !== 'string') { + throw new TypeError('encoding must be a string') + } + if (typeof encoding === 'string' && !Buffer.isEncoding(encoding)) { + throw new TypeError('Unknown encoding: ' + encoding) + } + } else if (typeof val === 'number') { + val = val & 255; + } + + // Invalid ranges are not set to a default, so can range check early. + if (start < 0 || this.length < start || this.length < end) { + throw new RangeError('Out of range index') + } + + if (end <= start) { + return this + } + + start = start >>> 0; + end = end === undefined ? this.length : end >>> 0; + + if (!val) val = 0; + + var i; + if (typeof val === 'number') { + for (i = start; i < end; ++i) { + this[i] = val; + } + } else { + var bytes = internalIsBuffer(val) + ? val + : utf8ToBytes(new Buffer(val, encoding).toString()); + var len = bytes.length; + for (i = 0; i < end - start; ++i) { + this[i + start] = bytes[i % len]; + } + } + + return this + }; + + // HELPER FUNCTIONS + // ================ + + var INVALID_BASE64_RE = /[^+\/0-9A-Za-z-_]/g; + + function base64clean (str) { + // Node strips out invalid characters like \n and \t from the string, base64-js does not + str = stringtrim(str).replace(INVALID_BASE64_RE, ''); + // Node converts strings with length < 2 to '' + if (str.length < 2) return '' + // Node allows for non-padded base64 strings (missing trailing ===), base64-js does not + while (str.length % 4 !== 0) { + str = str + '='; + } + return str + } + + function stringtrim (str) { + if (str.trim) return str.trim() + return str.replace(/^\s+|\s+$/g, '') + } + + function toHex (n) { + if (n < 16) return '0' + n.toString(16) + return n.toString(16) + } + + function utf8ToBytes (string, units) { + units = units || Infinity; + var codePoint; + var length = string.length; + var leadSurrogate = null; + var bytes = []; + + for (var i = 0; i < length; ++i) { + codePoint = string.charCodeAt(i); + + // is surrogate component + if (codePoint > 0xD7FF && codePoint < 0xE000) { + // last char was a lead + if (!leadSurrogate) { + // no lead yet + if (codePoint > 0xDBFF) { + // unexpected trail + if ((units -= 3) > -1) bytes.push(0xEF, 0xBF, 0xBD); + continue + } else if (i + 1 === length) { + // unpaired lead + if ((units -= 3) > -1) bytes.push(0xEF, 0xBF, 0xBD); + continue + } + + // valid lead + leadSurrogate = codePoint; + + continue + } + + // 2 leads in a row + if (codePoint < 0xDC00) { + if ((units -= 3) > -1) bytes.push(0xEF, 0xBF, 0xBD); + leadSurrogate = codePoint; + continue + } + + // valid surrogate pair + codePoint = (leadSurrogate - 0xD800 << 10 | codePoint - 0xDC00) + 0x10000; + } else if (leadSurrogate) { + // valid bmp char, but last char was a lead + if ((units -= 3) > -1) bytes.push(0xEF, 0xBF, 0xBD); + } + + leadSurrogate = null; + + // encode utf8 + if (codePoint < 0x80) { + if ((units -= 1) < 0) break + bytes.push(codePoint); + } else if (codePoint < 0x800) { + if ((units -= 2) < 0) break + bytes.push( + codePoint >> 0x6 | 0xC0, + codePoint & 0x3F | 0x80 + ); + } else if (codePoint < 0x10000) { + if ((units -= 3) < 0) break + bytes.push( + codePoint >> 0xC | 0xE0, + codePoint >> 0x6 & 0x3F | 0x80, + codePoint & 0x3F | 0x80 + ); + } else if (codePoint < 0x110000) { + if ((units -= 4) < 0) break + bytes.push( + codePoint >> 0x12 | 0xF0, + codePoint >> 0xC & 0x3F | 0x80, + codePoint >> 0x6 & 0x3F | 0x80, + codePoint & 0x3F | 0x80 + ); + } else { + throw new Error('Invalid code point') + } + } + + return bytes + } + + function asciiToBytes (str) { + var byteArray = []; + for (var i = 0; i < str.length; ++i) { + // Node's code seems to be doing this and not & 0x7F.. + byteArray.push(str.charCodeAt(i) & 0xFF); + } + return byteArray + } + + function utf16leToBytes (str, units) { + var c, hi, lo; + var byteArray = []; + for (var i = 0; i < str.length; ++i) { + if ((units -= 2) < 0) break + + c = str.charCodeAt(i); + hi = c >> 8; + lo = c % 256; + byteArray.push(lo); + byteArray.push(hi); + } + + return byteArray + } + + + function base64ToBytes (str) { + return toByteArray(base64clean(str)) + } + + function blitBuffer (src, dst, offset, length) { + for (var i = 0; i < length; ++i) { + if ((i + offset >= dst.length) || (i >= src.length)) break + dst[i + offset] = src[i]; + } + return i + } + + function isnan (val) { + return val !== val // eslint-disable-line no-self-compare + } + + + // the following is from is-buffer, also by Feross Aboukhadijeh and with same lisence + // The _isBuffer check is for Safari 5-7 support, because it's missing + // Object.prototype.constructor. Remove this eventually + function isBuffer(obj) { + return obj != null && (!!obj._isBuffer || isFastBuffer(obj) || isSlowBuffer(obj)) + } + + function isFastBuffer (obj) { + return !!obj.constructor && typeof obj.constructor.isBuffer === 'function' && obj.constructor.isBuffer(obj) + } + + // For Node v0.10 support. Remove this eventually. + function isSlowBuffer (obj) { + return typeof obj.readFloatLE === 'function' && typeof obj.slice === 'function' && isFastBuffer(obj.slice(0, 0)) + } + + // shim for using process in browser + // based off https://github.com/defunctzombie/node-process/blob/master/browser.js + + function defaultSetTimout() { + throw new Error('setTimeout has not been defined'); + } + function defaultClearTimeout () { + throw new Error('clearTimeout has not been defined'); + } + var cachedSetTimeout = defaultSetTimout; + var cachedClearTimeout = defaultClearTimeout; + if (typeof global$1.setTimeout === 'function') { + cachedSetTimeout = setTimeout; + } + if (typeof global$1.clearTimeout === 'function') { + cachedClearTimeout = clearTimeout; + } + + function runTimeout(fun) { + if (cachedSetTimeout === setTimeout) { + //normal enviroments in sane situations + return setTimeout(fun, 0); + } + // if setTimeout wasn't available but was latter defined + if ((cachedSetTimeout === defaultSetTimout || !cachedSetTimeout) && setTimeout) { + cachedSetTimeout = setTimeout; + return setTimeout(fun, 0); + } + try { + // when when somebody has screwed with setTimeout but no I.E. maddness + return cachedSetTimeout(fun, 0); + } catch(e){ + try { + // When we are in I.E. but the script has been evaled so I.E. doesn't trust the global object when called normally + return cachedSetTimeout.call(null, fun, 0); + } catch(e){ + // same as above but when it's a version of I.E. that must have the global object for 'this', hopfully our context correct otherwise it will throw a global error + return cachedSetTimeout.call(this, fun, 0); + } + } + + + } + function runClearTimeout(marker) { + if (cachedClearTimeout === clearTimeout) { + //normal enviroments in sane situations + return clearTimeout(marker); + } + // if clearTimeout wasn't available but was latter defined + if ((cachedClearTimeout === defaultClearTimeout || !cachedClearTimeout) && clearTimeout) { + cachedClearTimeout = clearTimeout; + return clearTimeout(marker); + } + try { + // when when somebody has screwed with setTimeout but no I.E. maddness + return cachedClearTimeout(marker); + } catch (e){ + try { + // When we are in I.E. but the script has been evaled so I.E. doesn't trust the global object when called normally + return cachedClearTimeout.call(null, marker); + } catch (e){ + // same as above but when it's a version of I.E. that must have the global object for 'this', hopfully our context correct otherwise it will throw a global error. + // Some versions of I.E. have different rules for clearTimeout vs setTimeout + return cachedClearTimeout.call(this, marker); + } + } + + + + } + var queue = []; + var draining = false; + var currentQueue; + var queueIndex = -1; + + function cleanUpNextTick() { + if (!draining || !currentQueue) { + return; + } + draining = false; + if (currentQueue.length) { + queue = currentQueue.concat(queue); + } else { + queueIndex = -1; + } + if (queue.length) { + drainQueue(); + } + } + + function drainQueue() { + if (draining) { + return; + } + var timeout = runTimeout(cleanUpNextTick); + draining = true; + + var len = queue.length; + while(len) { + currentQueue = queue; + queue = []; + while (++queueIndex < len) { + if (currentQueue) { + currentQueue[queueIndex].run(); + } + } + queueIndex = -1; + len = queue.length; + } + currentQueue = null; + draining = false; + runClearTimeout(timeout); + } + function nextTick(fun) { + var args = new Array(arguments.length - 1); + if (arguments.length > 1) { + for (var i = 1; i < arguments.length; i++) { + args[i - 1] = arguments[i]; + } + } + queue.push(new Item(fun, args)); + if (queue.length === 1 && !draining) { + runTimeout(drainQueue); + } + } + // v8 likes predictible objects + function Item(fun, array) { + this.fun = fun; + this.array = array; + } + Item.prototype.run = function () { + this.fun.apply(null, this.array); + }; + + // from https://github.com/kumavis/browser-process-hrtime/blob/master/index.js + var performance = global$1.performance || {}; + performance.now || + performance.mozNow || + performance.msNow || + performance.oNow || + performance.webkitNow || + function(){ return (new Date()).getTime() }; + + var inherits; + if (typeof Object.create === 'function'){ + inherits = function inherits(ctor, superCtor) { + // implementation from standard node.js 'util' module + ctor.super_ = superCtor; + ctor.prototype = Object.create(superCtor.prototype, { + constructor: { + value: ctor, + enumerable: false, + writable: true, + configurable: true + } + }); + }; + } else { + inherits = function inherits(ctor, superCtor) { + ctor.super_ = superCtor; + var TempCtor = function () {}; + TempCtor.prototype = superCtor.prototype; + ctor.prototype = new TempCtor(); + ctor.prototype.constructor = ctor; + }; + } + var inherits$1 = inherits; + + var formatRegExp = /%[sdj%]/g; + function format(f) { + if (!isString(f)) { + var objects = []; + for (var i = 0; i < arguments.length; i++) { + objects.push(inspect(arguments[i])); + } + return objects.join(' '); + } + + var i = 1; + var args = arguments; + var len = args.length; + var str = String(f).replace(formatRegExp, function(x) { + if (x === '%%') return '%'; + if (i >= len) return x; + switch (x) { + case '%s': return String(args[i++]); + case '%d': return Number(args[i++]); + case '%j': + try { + return JSON.stringify(args[i++]); + } catch (_) { + return '[Circular]'; + } + default: + return x; + } + }); + for (var x = args[i]; i < len; x = args[++i]) { + if (isNull(x) || !isObject(x)) { + str += ' ' + x; + } else { + str += ' ' + inspect(x); + } + } + return str; + } + + // Mark that a method should not be used. + // Returns a modified function which warns once by default. + // If --no-deprecation is set, then it is a no-op. + function deprecate(fn, msg) { + // Allow for deprecating things in the process of starting up. + if (isUndefined(global$1.process)) { + return function() { + return deprecate(fn, msg).apply(this, arguments); + }; + } + + var warned = false; + function deprecated() { + if (!warned) { + { + console.error(msg); + } + warned = true; + } + return fn.apply(this, arguments); + } + + return deprecated; + } + + var debugs = {}; + var debugEnviron; + function debuglog(set) { + if (isUndefined(debugEnviron)) + debugEnviron = ''; + set = set.toUpperCase(); + if (!debugs[set]) { + if (new RegExp('\\b' + set + '\\b', 'i').test(debugEnviron)) { + var pid = 0; + debugs[set] = function() { + var msg = format.apply(null, arguments); + console.error('%s %d: %s', set, pid, msg); + }; + } else { + debugs[set] = function() {}; + } + } + return debugs[set]; + } + + /** + * Echos the value of a value. Trys to print the value out + * in the best way possible given the different types. + * + * @param {Object} obj The object to print out. + * @param {Object} opts Optional options object that alters the output. + */ + /* legacy: obj, showHidden, depth, colors*/ + function inspect(obj, opts) { + // default options + var ctx = { + seen: [], + stylize: stylizeNoColor + }; + // legacy... + if (arguments.length >= 3) ctx.depth = arguments[2]; + if (arguments.length >= 4) ctx.colors = arguments[3]; + if (isBoolean(opts)) { + // legacy... + ctx.showHidden = opts; + } else if (opts) { + // got an "options" object + _extend(ctx, opts); + } + // set default options + if (isUndefined(ctx.showHidden)) ctx.showHidden = false; + if (isUndefined(ctx.depth)) ctx.depth = 2; + if (isUndefined(ctx.colors)) ctx.colors = false; + if (isUndefined(ctx.customInspect)) ctx.customInspect = true; + if (ctx.colors) ctx.stylize = stylizeWithColor; + return formatValue(ctx, obj, ctx.depth); + } + + // http://en.wikipedia.org/wiki/ANSI_escape_code#graphics + inspect.colors = { + 'bold' : [1, 22], + 'italic' : [3, 23], + 'underline' : [4, 24], + 'inverse' : [7, 27], + 'white' : [37, 39], + 'grey' : [90, 39], + 'black' : [30, 39], + 'blue' : [34, 39], + 'cyan' : [36, 39], + 'green' : [32, 39], + 'magenta' : [35, 39], + 'red' : [31, 39], + 'yellow' : [33, 39] + }; + + // Don't use 'blue' not visible on cmd.exe + inspect.styles = { + 'special': 'cyan', + 'number': 'yellow', + 'boolean': 'yellow', + 'undefined': 'grey', + 'null': 'bold', + 'string': 'green', + 'date': 'magenta', + // "name": intentionally not styling + 'regexp': 'red' + }; + + + function stylizeWithColor(str, styleType) { + var style = inspect.styles[styleType]; + + if (style) { + return '\u001b[' + inspect.colors[style][0] + 'm' + str + + '\u001b[' + inspect.colors[style][1] + 'm'; + } else { + return str; + } + } + + + function stylizeNoColor(str, styleType) { + return str; + } + + + function arrayToHash(array) { + var hash = {}; + + array.forEach(function(val, idx) { + hash[val] = true; + }); + + return hash; + } + + + function formatValue(ctx, value, recurseTimes) { + // Provide a hook for user-specified inspect functions. + // Check that value is an object with an inspect function on it + if (ctx.customInspect && + value && + isFunction(value.inspect) && + // Filter out the util module, it's inspect function is special + value.inspect !== inspect && + // Also filter out any prototype objects using the circular check. + !(value.constructor && value.constructor.prototype === value)) { + var ret = value.inspect(recurseTimes, ctx); + if (!isString(ret)) { + ret = formatValue(ctx, ret, recurseTimes); + } + return ret; + } + + // Primitive types cannot have properties + var primitive = formatPrimitive(ctx, value); + if (primitive) { + return primitive; + } + + // Look up the keys of the object. + var keys = Object.keys(value); + var visibleKeys = arrayToHash(keys); + + if (ctx.showHidden) { + keys = Object.getOwnPropertyNames(value); + } + + // IE doesn't make error fields non-enumerable + // http://msdn.microsoft.com/en-us/library/ie/dww52sbt(v=vs.94).aspx + if (isError(value) + && (keys.indexOf('message') >= 0 || keys.indexOf('description') >= 0)) { + return formatError(value); + } + + // Some type of object without properties can be shortcutted. + if (keys.length === 0) { + if (isFunction(value)) { + var name = value.name ? ': ' + value.name : ''; + return ctx.stylize('[Function' + name + ']', 'special'); + } + if (isRegExp(value)) { + return ctx.stylize(RegExp.prototype.toString.call(value), 'regexp'); + } + if (isDate(value)) { + return ctx.stylize(Date.prototype.toString.call(value), 'date'); + } + if (isError(value)) { + return formatError(value); + } + } + + var base = '', array = false, braces = ['{', '}']; + + // Make Array say that they are Array + if (isArray(value)) { + array = true; + braces = ['[', ']']; + } + + // Make functions say that they are functions + if (isFunction(value)) { + var n = value.name ? ': ' + value.name : ''; + base = ' [Function' + n + ']'; + } + + // Make RegExps say that they are RegExps + if (isRegExp(value)) { + base = ' ' + RegExp.prototype.toString.call(value); + } + + // Make dates with properties first say the date + if (isDate(value)) { + base = ' ' + Date.prototype.toUTCString.call(value); + } + + // Make error with message first say the error + if (isError(value)) { + base = ' ' + formatError(value); + } + + if (keys.length === 0 && (!array || value.length == 0)) { + return braces[0] + base + braces[1]; + } + + if (recurseTimes < 0) { + if (isRegExp(value)) { + return ctx.stylize(RegExp.prototype.toString.call(value), 'regexp'); + } else { + return ctx.stylize('[Object]', 'special'); + } + } + + ctx.seen.push(value); + + var output; + if (array) { + output = formatArray(ctx, value, recurseTimes, visibleKeys, keys); + } else { + output = keys.map(function(key) { + return formatProperty(ctx, value, recurseTimes, visibleKeys, key, array); + }); + } + + ctx.seen.pop(); + + return reduceToSingleString(output, base, braces); + } + + + function formatPrimitive(ctx, value) { + if (isUndefined(value)) + return ctx.stylize('undefined', 'undefined'); + if (isString(value)) { + var simple = '\'' + JSON.stringify(value).replace(/^"|"$/g, '') + .replace(/'/g, "\\'") + .replace(/\\"/g, '"') + '\''; + return ctx.stylize(simple, 'string'); + } + if (isNumber(value)) + return ctx.stylize('' + value, 'number'); + if (isBoolean(value)) + return ctx.stylize('' + value, 'boolean'); + // For some reason typeof null is "object", so special case here. + if (isNull(value)) + return ctx.stylize('null', 'null'); + } + + + function formatError(value) { + return '[' + Error.prototype.toString.call(value) + ']'; + } + + + function formatArray(ctx, value, recurseTimes, visibleKeys, keys) { + var output = []; + for (var i = 0, l = value.length; i < l; ++i) { + if (hasOwnProperty(value, String(i))) { + output.push(formatProperty(ctx, value, recurseTimes, visibleKeys, + String(i), true)); + } else { + output.push(''); + } + } + keys.forEach(function(key) { + if (!key.match(/^\d+$/)) { + output.push(formatProperty(ctx, value, recurseTimes, visibleKeys, + key, true)); + } + }); + return output; + } + + + function formatProperty(ctx, value, recurseTimes, visibleKeys, key, array) { + var name, str, desc; + desc = Object.getOwnPropertyDescriptor(value, key) || { value: value[key] }; + if (desc.get) { + if (desc.set) { + str = ctx.stylize('[Getter/Setter]', 'special'); + } else { + str = ctx.stylize('[Getter]', 'special'); + } + } else { + if (desc.set) { + str = ctx.stylize('[Setter]', 'special'); + } + } + if (!hasOwnProperty(visibleKeys, key)) { + name = '[' + key + ']'; + } + if (!str) { + if (ctx.seen.indexOf(desc.value) < 0) { + if (isNull(recurseTimes)) { + str = formatValue(ctx, desc.value, null); + } else { + str = formatValue(ctx, desc.value, recurseTimes - 1); + } + if (str.indexOf('\n') > -1) { + if (array) { + str = str.split('\n').map(function(line) { + return ' ' + line; + }).join('\n').substr(2); + } else { + str = '\n' + str.split('\n').map(function(line) { + return ' ' + line; + }).join('\n'); + } + } + } else { + str = ctx.stylize('[Circular]', 'special'); + } + } + if (isUndefined(name)) { + if (array && key.match(/^\d+$/)) { + return str; + } + name = JSON.stringify('' + key); + if (name.match(/^"([a-zA-Z_][a-zA-Z_0-9]*)"$/)) { + name = name.substr(1, name.length - 2); + name = ctx.stylize(name, 'name'); + } else { + name = name.replace(/'/g, "\\'") + .replace(/\\"/g, '"') + .replace(/(^"|"$)/g, "'"); + name = ctx.stylize(name, 'string'); + } + } + + return name + ': ' + str; + } + + + function reduceToSingleString(output, base, braces) { + var length = output.reduce(function(prev, cur) { + if (cur.indexOf('\n') >= 0) ; + return prev + cur.replace(/\u001b\[\d\d?m/g, '').length + 1; + }, 0); + + if (length > 60) { + return braces[0] + + (base === '' ? '' : base + '\n ') + + ' ' + + output.join(',\n ') + + ' ' + + braces[1]; + } + + return braces[0] + base + ' ' + output.join(', ') + ' ' + braces[1]; + } + + + // NOTE: These type checking functions intentionally don't use `instanceof` + // because it is fragile and can be easily faked with `Object.create()`. + function isArray(ar) { + return Array.isArray(ar); + } + + function isBoolean(arg) { + return typeof arg === 'boolean'; + } + + function isNull(arg) { + return arg === null; + } + + function isNumber(arg) { + return typeof arg === 'number'; + } + + function isString(arg) { + return typeof arg === 'string'; + } + + function isUndefined(arg) { + return arg === void 0; + } + + function isRegExp(re) { + return isObject(re) && objectToString(re) === '[object RegExp]'; + } + + function isObject(arg) { + return typeof arg === 'object' && arg !== null; + } + + function isDate(d) { + return isObject(d) && objectToString(d) === '[object Date]'; + } + + function isError(e) { + return isObject(e) && + (objectToString(e) === '[object Error]' || e instanceof Error); + } + + function isFunction(arg) { + return typeof arg === 'function'; + } + + function objectToString(o) { + return Object.prototype.toString.call(o); + } + + function _extend(origin, add) { + // Don't do anything if add isn't an object + if (!add || !isObject(add)) return origin; + + var keys = Object.keys(add); + var i = keys.length; + while (i--) { + origin[keys[i]] = add[keys[i]]; + } + return origin; + } + function hasOwnProperty(obj, prop) { + return Object.prototype.hasOwnProperty.call(obj, prop); + } + + function BufferList() { + this.head = null; + this.tail = null; + this.length = 0; + } + + BufferList.prototype.push = function (v) { + var entry = { data: v, next: null }; + if (this.length > 0) this.tail.next = entry;else this.head = entry; + this.tail = entry; + ++this.length; + }; + + BufferList.prototype.unshift = function (v) { + var entry = { data: v, next: this.head }; + if (this.length === 0) this.tail = entry; + this.head = entry; + ++this.length; + }; + + BufferList.prototype.shift = function () { + if (this.length === 0) return; + var ret = this.head.data; + if (this.length === 1) this.head = this.tail = null;else this.head = this.head.next; + --this.length; + return ret; + }; + + BufferList.prototype.clear = function () { + this.head = this.tail = null; + this.length = 0; + }; + + BufferList.prototype.join = function (s) { + if (this.length === 0) return ''; + var p = this.head; + var ret = '' + p.data; + while (p = p.next) { + ret += s + p.data; + }return ret; + }; + + BufferList.prototype.concat = function (n) { + if (this.length === 0) return Buffer.alloc(0); + if (this.length === 1) return this.head.data; + var ret = Buffer.allocUnsafe(n >>> 0); + var p = this.head; + var i = 0; + while (p) { + p.data.copy(ret, i); + i += p.data.length; + p = p.next; + } + return ret; + }; + + // Copyright Joyent, Inc. and other Node contributors. + var isBufferEncoding = Buffer.isEncoding + || function(encoding) { + switch (encoding && encoding.toLowerCase()) { + case 'hex': case 'utf8': case 'utf-8': case 'ascii': case 'binary': case 'base64': case 'ucs2': case 'ucs-2': case 'utf16le': case 'utf-16le': case 'raw': return true; + default: return false; + } + }; + + + function assertEncoding(encoding) { + if (encoding && !isBufferEncoding(encoding)) { + throw new Error('Unknown encoding: ' + encoding); + } + } + + // StringDecoder provides an interface for efficiently splitting a series of + // buffers into a series of JS strings without breaking apart multi-byte + // characters. CESU-8 is handled as part of the UTF-8 encoding. + // + // @TODO Handling all encodings inside a single object makes it very difficult + // to reason about this code, so it should be split up in the future. + // @TODO There should be a utf8-strict encoding that rejects invalid UTF-8 code + // points as used by CESU-8. + function StringDecoder(encoding) { + this.encoding = (encoding || 'utf8').toLowerCase().replace(/[-_]/, ''); + assertEncoding(encoding); + switch (this.encoding) { + case 'utf8': + // CESU-8 represents each of Surrogate Pair by 3-bytes + this.surrogateSize = 3; + break; + case 'ucs2': + case 'utf16le': + // UTF-16 represents each of Surrogate Pair by 2-bytes + this.surrogateSize = 2; + this.detectIncompleteChar = utf16DetectIncompleteChar; + break; + case 'base64': + // Base-64 stores 3 bytes in 4 chars, and pads the remainder. + this.surrogateSize = 3; + this.detectIncompleteChar = base64DetectIncompleteChar; + break; + default: + this.write = passThroughWrite; + return; + } + + // Enough space to store all bytes of a single character. UTF-8 needs 4 + // bytes, but CESU-8 may require up to 6 (3 bytes per surrogate). + this.charBuffer = new Buffer(6); + // Number of bytes received for the current incomplete multi-byte character. + this.charReceived = 0; + // Number of bytes expected for the current incomplete multi-byte character. + this.charLength = 0; + } + + // write decodes the given buffer and returns it as JS string that is + // guaranteed to not contain any partial multi-byte characters. Any partial + // character found at the end of the buffer is buffered up, and will be + // returned when calling write again with the remaining bytes. + // + // Note: Converting a Buffer containing an orphan surrogate to a String + // currently works, but converting a String to a Buffer (via `new Buffer`, or + // Buffer#write) will replace incomplete surrogates with the unicode + // replacement character. See https://codereview.chromium.org/121173009/ . + StringDecoder.prototype.write = function(buffer) { + var charStr = ''; + // if our last write ended with an incomplete multibyte character + while (this.charLength) { + // determine how many remaining bytes this buffer has to offer for this char + var available = (buffer.length >= this.charLength - this.charReceived) ? + this.charLength - this.charReceived : + buffer.length; + + // add the new bytes to the char buffer + buffer.copy(this.charBuffer, this.charReceived, 0, available); + this.charReceived += available; + + if (this.charReceived < this.charLength) { + // still not enough chars in this buffer? wait for more ... + return ''; + } + + // remove bytes belonging to the current character from the buffer + buffer = buffer.slice(available, buffer.length); + + // get the character that was split + charStr = this.charBuffer.slice(0, this.charLength).toString(this.encoding); + + // CESU-8: lead surrogate (D800-DBFF) is also the incomplete character + var charCode = charStr.charCodeAt(charStr.length - 1); + if (charCode >= 0xD800 && charCode <= 0xDBFF) { + this.charLength += this.surrogateSize; + charStr = ''; + continue; + } + this.charReceived = this.charLength = 0; + + // if there are no more bytes in this buffer, just emit our char + if (buffer.length === 0) { + return charStr; + } + break; + } + + // determine and set charLength / charReceived + this.detectIncompleteChar(buffer); + + var end = buffer.length; + if (this.charLength) { + // buffer the incomplete character bytes we got + buffer.copy(this.charBuffer, 0, buffer.length - this.charReceived, end); + end -= this.charReceived; + } + + charStr += buffer.toString(this.encoding, 0, end); + + var end = charStr.length - 1; + var charCode = charStr.charCodeAt(end); + // CESU-8: lead surrogate (D800-DBFF) is also the incomplete character + if (charCode >= 0xD800 && charCode <= 0xDBFF) { + var size = this.surrogateSize; + this.charLength += size; + this.charReceived += size; + this.charBuffer.copy(this.charBuffer, size, 0, size); + buffer.copy(this.charBuffer, 0, 0, size); + return charStr.substring(0, end); + } + + // or just emit the charStr + return charStr; + }; + + // detectIncompleteChar determines if there is an incomplete UTF-8 character at + // the end of the given buffer. If so, it sets this.charLength to the byte + // length that character, and sets this.charReceived to the number of bytes + // that are available for this character. + StringDecoder.prototype.detectIncompleteChar = function(buffer) { + // determine how many bytes we have to check at the end of this buffer + var i = (buffer.length >= 3) ? 3 : buffer.length; + + // Figure out if one of the last i bytes of our buffer announces an + // incomplete char. + for (; i > 0; i--) { + var c = buffer[buffer.length - i]; + + // See http://en.wikipedia.org/wiki/UTF-8#Description + + // 110XXXXX + if (i == 1 && c >> 5 == 0x06) { + this.charLength = 2; + break; + } + + // 1110XXXX + if (i <= 2 && c >> 4 == 0x0E) { + this.charLength = 3; + break; + } + + // 11110XXX + if (i <= 3 && c >> 3 == 0x1E) { + this.charLength = 4; + break; + } + } + this.charReceived = i; + }; + + StringDecoder.prototype.end = function(buffer) { + var res = ''; + if (buffer && buffer.length) + res = this.write(buffer); + + if (this.charReceived) { + var cr = this.charReceived; + var buf = this.charBuffer; + var enc = this.encoding; + res += buf.slice(0, cr).toString(enc); + } + + return res; + }; + + function passThroughWrite(buffer) { + return buffer.toString(this.encoding); + } + + function utf16DetectIncompleteChar(buffer) { + this.charReceived = buffer.length % 2; + this.charLength = this.charReceived ? 2 : 0; + } + + function base64DetectIncompleteChar(buffer) { + this.charReceived = buffer.length % 3; + this.charLength = this.charReceived ? 3 : 0; + } + + Readable.ReadableState = ReadableState; + + var debug = debuglog('stream'); + inherits$1(Readable, EventEmitter); + + function prependListener(emitter, event, fn) { + // Sadly this is not cacheable as some libraries bundle their own + // event emitter implementation with them. + if (typeof emitter.prependListener === 'function') { + return emitter.prependListener(event, fn); + } else { + // This is a hack to make sure that our error handler is attached before any + // userland ones. NEVER DO THIS. This is here only because this code needs + // to continue to work with older versions of Node.js that do not include + // the prependListener() method. The goal is to eventually remove this hack. + if (!emitter._events || !emitter._events[event]) + emitter.on(event, fn); + else if (Array.isArray(emitter._events[event])) + emitter._events[event].unshift(fn); + else + emitter._events[event] = [fn, emitter._events[event]]; + } + } + function listenerCount (emitter, type) { + return emitter.listeners(type).length; + } + function ReadableState(options, stream) { + + options = options || {}; + + // object stream flag. Used to make read(n) ignore n and to + // make all the buffer merging and length checks go away + this.objectMode = !!options.objectMode; + + if (stream instanceof Duplex) this.objectMode = this.objectMode || !!options.readableObjectMode; + + // the point at which it stops calling _read() to fill the buffer + // Note: 0 is a valid value, means "don't call _read preemptively ever" + var hwm = options.highWaterMark; + var defaultHwm = this.objectMode ? 16 : 16 * 1024; + this.highWaterMark = hwm || hwm === 0 ? hwm : defaultHwm; + + // cast to ints. + this.highWaterMark = ~ ~this.highWaterMark; + + // A linked list is used to store data chunks instead of an array because the + // linked list can remove elements from the beginning faster than + // array.shift() + this.buffer = new BufferList(); + this.length = 0; + this.pipes = null; + this.pipesCount = 0; + this.flowing = null; + this.ended = false; + this.endEmitted = false; + this.reading = false; + + // a flag to be able to tell if the onwrite cb is called immediately, + // or on a later tick. We set this to true at first, because any + // actions that shouldn't happen until "later" should generally also + // not happen before the first write call. + this.sync = true; + + // whenever we return null, then we set a flag to say + // that we're awaiting a 'readable' event emission. + this.needReadable = false; + this.emittedReadable = false; + this.readableListening = false; + this.resumeScheduled = false; + + // Crypto is kind of old and crusty. Historically, its default string + // encoding is 'binary' so we have to make this configurable. + // Everything else in the universe uses 'utf8', though. + this.defaultEncoding = options.defaultEncoding || 'utf8'; + + // when piping, we only care about 'readable' events that happen + // after read()ing all the bytes and not getting any pushback. + this.ranOut = false; + + // the number of writers that are awaiting a drain event in .pipe()s + this.awaitDrain = 0; + + // if true, a maybeReadMore has been scheduled + this.readingMore = false; + + this.decoder = null; + this.encoding = null; + if (options.encoding) { + this.decoder = new StringDecoder(options.encoding); + this.encoding = options.encoding; + } + } + function Readable(options) { + + if (!(this instanceof Readable)) return new Readable(options); + + this._readableState = new ReadableState(options, this); + + // legacy + this.readable = true; + + if (options && typeof options.read === 'function') this._read = options.read; + + EventEmitter.call(this); + } + + // Manually shove something into the read() buffer. + // This returns true if the highWaterMark has not been hit yet, + // similar to how Writable.write() returns true if you should + // write() some more. + Readable.prototype.push = function (chunk, encoding) { + var state = this._readableState; + + if (!state.objectMode && typeof chunk === 'string') { + encoding = encoding || state.defaultEncoding; + if (encoding !== state.encoding) { + chunk = Buffer.from(chunk, encoding); + encoding = ''; + } + } + + return readableAddChunk(this, state, chunk, encoding, false); + }; + + // Unshift should *always* be something directly out of read() + Readable.prototype.unshift = function (chunk) { + var state = this._readableState; + return readableAddChunk(this, state, chunk, '', true); + }; + + Readable.prototype.isPaused = function () { + return this._readableState.flowing === false; + }; + + function readableAddChunk(stream, state, chunk, encoding, addToFront) { + var er = chunkInvalid(state, chunk); + if (er) { + stream.emit('error', er); + } else if (chunk === null) { + state.reading = false; + onEofChunk(stream, state); + } else if (state.objectMode || chunk && chunk.length > 0) { + if (state.ended && !addToFront) { + var e = new Error('stream.push() after EOF'); + stream.emit('error', e); + } else if (state.endEmitted && addToFront) { + var _e = new Error('stream.unshift() after end event'); + stream.emit('error', _e); + } else { + var skipAdd; + if (state.decoder && !addToFront && !encoding) { + chunk = state.decoder.write(chunk); + skipAdd = !state.objectMode && chunk.length === 0; + } + + if (!addToFront) state.reading = false; + + // Don't add to the buffer if we've decoded to an empty string chunk and + // we're not in object mode + if (!skipAdd) { + // if we want the data now, just emit it. + if (state.flowing && state.length === 0 && !state.sync) { + stream.emit('data', chunk); + stream.read(0); + } else { + // update the buffer info. + state.length += state.objectMode ? 1 : chunk.length; + if (addToFront) state.buffer.unshift(chunk);else state.buffer.push(chunk); + + if (state.needReadable) emitReadable(stream); + } + } + + maybeReadMore(stream, state); + } + } else if (!addToFront) { + state.reading = false; + } + + return needMoreData(state); + } + + // if it's past the high water mark, we can push in some more. + // Also, if we have no data yet, we can stand some + // more bytes. This is to work around cases where hwm=0, + // such as the repl. Also, if the push() triggered a + // readable event, and the user called read(largeNumber) such that + // needReadable was set, then we ought to push more, so that another + // 'readable' event will be triggered. + function needMoreData(state) { + return !state.ended && (state.needReadable || state.length < state.highWaterMark || state.length === 0); + } + + // backwards compatibility. + Readable.prototype.setEncoding = function (enc) { + this._readableState.decoder = new StringDecoder(enc); + this._readableState.encoding = enc; + return this; + }; + + // Don't raise the hwm > 8MB + var MAX_HWM = 0x800000; + function computeNewHighWaterMark(n) { + if (n >= MAX_HWM) { + n = MAX_HWM; + } else { + // Get the next highest power of 2 to prevent increasing hwm excessively in + // tiny amounts + n--; + n |= n >>> 1; + n |= n >>> 2; + n |= n >>> 4; + n |= n >>> 8; + n |= n >>> 16; + n++; + } + return n; + } + + // This function is designed to be inlinable, so please take care when making + // changes to the function body. + function howMuchToRead(n, state) { + if (n <= 0 || state.length === 0 && state.ended) return 0; + if (state.objectMode) return 1; + if (n !== n) { + // Only flow one buffer at a time + if (state.flowing && state.length) return state.buffer.head.data.length;else return state.length; + } + // If we're asking for more than the current hwm, then raise the hwm. + if (n > state.highWaterMark) state.highWaterMark = computeNewHighWaterMark(n); + if (n <= state.length) return n; + // Don't have enough + if (!state.ended) { + state.needReadable = true; + return 0; + } + return state.length; + } + + // you can override either this method, or the async _read(n) below. + Readable.prototype.read = function (n) { + debug('read', n); + n = parseInt(n, 10); + var state = this._readableState; + var nOrig = n; + + if (n !== 0) state.emittedReadable = false; + + // if we're doing read(0) to trigger a readable event, but we + // already have a bunch of data in the buffer, then just trigger + // the 'readable' event and move on. + if (n === 0 && state.needReadable && (state.length >= state.highWaterMark || state.ended)) { + debug('read: emitReadable', state.length, state.ended); + if (state.length === 0 && state.ended) endReadable(this);else emitReadable(this); + return null; + } + + n = howMuchToRead(n, state); + + // if we've ended, and we're now clear, then finish it up. + if (n === 0 && state.ended) { + if (state.length === 0) endReadable(this); + return null; + } + + // All the actual chunk generation logic needs to be + // *below* the call to _read. The reason is that in certain + // synthetic stream cases, such as passthrough streams, _read + // may be a completely synchronous operation which may change + // the state of the read buffer, providing enough data when + // before there was *not* enough. + // + // So, the steps are: + // 1. Figure out what the state of things will be after we do + // a read from the buffer. + // + // 2. If that resulting state will trigger a _read, then call _read. + // Note that this may be asynchronous, or synchronous. Yes, it is + // deeply ugly to write APIs this way, but that still doesn't mean + // that the Readable class should behave improperly, as streams are + // designed to be sync/async agnostic. + // Take note if the _read call is sync or async (ie, if the read call + // has returned yet), so that we know whether or not it's safe to emit + // 'readable' etc. + // + // 3. Actually pull the requested chunks out of the buffer and return. + + // if we need a readable event, then we need to do some reading. + var doRead = state.needReadable; + debug('need readable', doRead); + + // if we currently have less than the highWaterMark, then also read some + if (state.length === 0 || state.length - n < state.highWaterMark) { + doRead = true; + debug('length less than watermark', doRead); + } + + // however, if we've ended, then there's no point, and if we're already + // reading, then it's unnecessary. + if (state.ended || state.reading) { + doRead = false; + debug('reading or ended', doRead); + } else if (doRead) { + debug('do read'); + state.reading = true; + state.sync = true; + // if the length is currently zero, then we *need* a readable event. + if (state.length === 0) state.needReadable = true; + // call internal read method + this._read(state.highWaterMark); + state.sync = false; + // If _read pushed data synchronously, then `reading` will be false, + // and we need to re-evaluate how much data we can return to the user. + if (!state.reading) n = howMuchToRead(nOrig, state); + } + + var ret; + if (n > 0) ret = fromList(n, state);else ret = null; + + if (ret === null) { + state.needReadable = true; + n = 0; + } else { + state.length -= n; + } + + if (state.length === 0) { + // If we have nothing in the buffer, then we want to know + // as soon as we *do* get something into the buffer. + if (!state.ended) state.needReadable = true; + + // If we tried to read() past the EOF, then emit end on the next tick. + if (nOrig !== n && state.ended) endReadable(this); + } + + if (ret !== null) this.emit('data', ret); + + return ret; + }; + + function chunkInvalid(state, chunk) { + var er = null; + if (!isBuffer(chunk) && typeof chunk !== 'string' && chunk !== null && chunk !== undefined && !state.objectMode) { + er = new TypeError('Invalid non-string/buffer chunk'); + } + return er; + } + + function onEofChunk(stream, state) { + if (state.ended) return; + if (state.decoder) { + var chunk = state.decoder.end(); + if (chunk && chunk.length) { + state.buffer.push(chunk); + state.length += state.objectMode ? 1 : chunk.length; + } + } + state.ended = true; + + // emit 'readable' now to make sure it gets picked up. + emitReadable(stream); + } + + // Don't emit readable right away in sync mode, because this can trigger + // another read() call => stack overflow. This way, it might trigger + // a nextTick recursion warning, but that's not so bad. + function emitReadable(stream) { + var state = stream._readableState; + state.needReadable = false; + if (!state.emittedReadable) { + debug('emitReadable', state.flowing); + state.emittedReadable = true; + if (state.sync) nextTick(emitReadable_, stream);else emitReadable_(stream); + } + } + + function emitReadable_(stream) { + debug('emit readable'); + stream.emit('readable'); + flow(stream); + } + + // at this point, the user has presumably seen the 'readable' event, + // and called read() to consume some data. that may have triggered + // in turn another _read(n) call, in which case reading = true if + // it's in progress. + // However, if we're not ended, or reading, and the length < hwm, + // then go ahead and try to read some more preemptively. + function maybeReadMore(stream, state) { + if (!state.readingMore) { + state.readingMore = true; + nextTick(maybeReadMore_, stream, state); + } + } + + function maybeReadMore_(stream, state) { + var len = state.length; + while (!state.reading && !state.flowing && !state.ended && state.length < state.highWaterMark) { + debug('maybeReadMore read 0'); + stream.read(0); + if (len === state.length) + // didn't get any data, stop spinning. + break;else len = state.length; + } + state.readingMore = false; + } + + // abstract method. to be overridden in specific implementation classes. + // call cb(er, data) where data is <= n in length. + // for virtual (non-string, non-buffer) streams, "length" is somewhat + // arbitrary, and perhaps not very meaningful. + Readable.prototype._read = function (n) { + this.emit('error', new Error('not implemented')); + }; + + Readable.prototype.pipe = function (dest, pipeOpts) { + var src = this; + var state = this._readableState; + + switch (state.pipesCount) { + case 0: + state.pipes = dest; + break; + case 1: + state.pipes = [state.pipes, dest]; + break; + default: + state.pipes.push(dest); + break; + } + state.pipesCount += 1; + debug('pipe count=%d opts=%j', state.pipesCount, pipeOpts); + + var doEnd = (!pipeOpts || pipeOpts.end !== false); + + var endFn = doEnd ? onend : cleanup; + if (state.endEmitted) nextTick(endFn);else src.once('end', endFn); + + dest.on('unpipe', onunpipe); + function onunpipe(readable) { + debug('onunpipe'); + if (readable === src) { + cleanup(); + } + } + + function onend() { + debug('onend'); + dest.end(); + } + + // when the dest drains, it reduces the awaitDrain counter + // on the source. This would be more elegant with a .once() + // handler in flow(), but adding and removing repeatedly is + // too slow. + var ondrain = pipeOnDrain(src); + dest.on('drain', ondrain); + + var cleanedUp = false; + function cleanup() { + debug('cleanup'); + // cleanup event handlers once the pipe is broken + dest.removeListener('close', onclose); + dest.removeListener('finish', onfinish); + dest.removeListener('drain', ondrain); + dest.removeListener('error', onerror); + dest.removeListener('unpipe', onunpipe); + src.removeListener('end', onend); + src.removeListener('end', cleanup); + src.removeListener('data', ondata); + + cleanedUp = true; + + // if the reader is waiting for a drain event from this + // specific writer, then it would cause it to never start + // flowing again. + // So, if this is awaiting a drain, then we just call it now. + // If we don't know, then assume that we are waiting for one. + if (state.awaitDrain && (!dest._writableState || dest._writableState.needDrain)) ondrain(); + } + + // If the user pushes more data while we're writing to dest then we'll end up + // in ondata again. However, we only want to increase awaitDrain once because + // dest will only emit one 'drain' event for the multiple writes. + // => Introduce a guard on increasing awaitDrain. + var increasedAwaitDrain = false; + src.on('data', ondata); + function ondata(chunk) { + debug('ondata'); + increasedAwaitDrain = false; + var ret = dest.write(chunk); + if (false === ret && !increasedAwaitDrain) { + // If the user unpiped during `dest.write()`, it is possible + // to get stuck in a permanently paused state if that write + // also returned false. + // => Check whether `dest` is still a piping destination. + if ((state.pipesCount === 1 && state.pipes === dest || state.pipesCount > 1 && indexOf(state.pipes, dest) !== -1) && !cleanedUp) { + debug('false write response, pause', src._readableState.awaitDrain); + src._readableState.awaitDrain++; + increasedAwaitDrain = true; + } + src.pause(); + } + } + + // if the dest has an error, then stop piping into it. + // however, don't suppress the throwing behavior for this. + function onerror(er) { + debug('onerror', er); + unpipe(); + dest.removeListener('error', onerror); + if (listenerCount(dest, 'error') === 0) dest.emit('error', er); + } + + // Make sure our error handler is attached before userland ones. + prependListener(dest, 'error', onerror); + + // Both close and finish should trigger unpipe, but only once. + function onclose() { + dest.removeListener('finish', onfinish); + unpipe(); + } + dest.once('close', onclose); + function onfinish() { + debug('onfinish'); + dest.removeListener('close', onclose); + unpipe(); + } + dest.once('finish', onfinish); + + function unpipe() { + debug('unpipe'); + src.unpipe(dest); + } + + // tell the dest that it's being piped to + dest.emit('pipe', src); + + // start the flow if it hasn't been started already. + if (!state.flowing) { + debug('pipe resume'); + src.resume(); + } + + return dest; + }; + + function pipeOnDrain(src) { + return function () { + var state = src._readableState; + debug('pipeOnDrain', state.awaitDrain); + if (state.awaitDrain) state.awaitDrain--; + if (state.awaitDrain === 0 && src.listeners('data').length) { + state.flowing = true; + flow(src); + } + }; + } + + Readable.prototype.unpipe = function (dest) { + var state = this._readableState; + + // if we're not piping anywhere, then do nothing. + if (state.pipesCount === 0) return this; + + // just one destination. most common case. + if (state.pipesCount === 1) { + // passed in one, but it's not the right one. + if (dest && dest !== state.pipes) return this; + + if (!dest) dest = state.pipes; + + // got a match. + state.pipes = null; + state.pipesCount = 0; + state.flowing = false; + if (dest) dest.emit('unpipe', this); + return this; + } + + // slow case. multiple pipe destinations. + + if (!dest) { + // remove all. + var dests = state.pipes; + var len = state.pipesCount; + state.pipes = null; + state.pipesCount = 0; + state.flowing = false; + + for (var _i = 0; _i < len; _i++) { + dests[_i].emit('unpipe', this); + }return this; + } + + // try to find the right one. + var i = indexOf(state.pipes, dest); + if (i === -1) return this; + + state.pipes.splice(i, 1); + state.pipesCount -= 1; + if (state.pipesCount === 1) state.pipes = state.pipes[0]; + + dest.emit('unpipe', this); + + return this; + }; + + // set up data events if they are asked for + // Ensure readable listeners eventually get something + Readable.prototype.on = function (ev, fn) { + var res = EventEmitter.prototype.on.call(this, ev, fn); + + if (ev === 'data') { + // Start flowing on next tick if stream isn't explicitly paused + if (this._readableState.flowing !== false) this.resume(); + } else if (ev === 'readable') { + var state = this._readableState; + if (!state.endEmitted && !state.readableListening) { + state.readableListening = state.needReadable = true; + state.emittedReadable = false; + if (!state.reading) { + nextTick(nReadingNextTick, this); + } else if (state.length) { + emitReadable(this); + } + } + } + + return res; + }; + Readable.prototype.addListener = Readable.prototype.on; + + function nReadingNextTick(self) { + debug('readable nexttick read 0'); + self.read(0); + } + + // pause() and resume() are remnants of the legacy readable stream API + // If the user uses them, then switch into old mode. + Readable.prototype.resume = function () { + var state = this._readableState; + if (!state.flowing) { + debug('resume'); + state.flowing = true; + resume(this, state); + } + return this; + }; + + function resume(stream, state) { + if (!state.resumeScheduled) { + state.resumeScheduled = true; + nextTick(resume_, stream, state); + } + } + + function resume_(stream, state) { + if (!state.reading) { + debug('resume read 0'); + stream.read(0); + } + + state.resumeScheduled = false; + state.awaitDrain = 0; + stream.emit('resume'); + flow(stream); + if (state.flowing && !state.reading) stream.read(0); + } + + Readable.prototype.pause = function () { + debug('call pause flowing=%j', this._readableState.flowing); + if (false !== this._readableState.flowing) { + debug('pause'); + this._readableState.flowing = false; + this.emit('pause'); + } + return this; + }; + + function flow(stream) { + var state = stream._readableState; + debug('flow', state.flowing); + while (state.flowing && stream.read() !== null) {} + } + + // wrap an old-style stream as the async data source. + // This is *not* part of the readable stream interface. + // It is an ugly unfortunate mess of history. + Readable.prototype.wrap = function (stream) { + var state = this._readableState; + var paused = false; + + var self = this; + stream.on('end', function () { + debug('wrapped end'); + if (state.decoder && !state.ended) { + var chunk = state.decoder.end(); + if (chunk && chunk.length) self.push(chunk); + } + + self.push(null); + }); + + stream.on('data', function (chunk) { + debug('wrapped data'); + if (state.decoder) chunk = state.decoder.write(chunk); + + // don't skip over falsy values in objectMode + if (state.objectMode && (chunk === null || chunk === undefined)) return;else if (!state.objectMode && (!chunk || !chunk.length)) return; + + var ret = self.push(chunk); + if (!ret) { + paused = true; + stream.pause(); + } + }); + + // proxy all the other methods. + // important when wrapping filters and duplexes. + for (var i in stream) { + if (this[i] === undefined && typeof stream[i] === 'function') { + this[i] = function (method) { + return function () { + return stream[method].apply(stream, arguments); + }; + }(i); + } + } + + // proxy certain important events. + var events = ['error', 'close', 'destroy', 'pause', 'resume']; + forEach(events, function (ev) { + stream.on(ev, self.emit.bind(self, ev)); + }); + + // when we try to consume some more bytes, simply unpause the + // underlying stream. + self._read = function (n) { + debug('wrapped _read', n); + if (paused) { + paused = false; + stream.resume(); + } + }; + + return self; + }; + + // exposed for testing purposes only. + Readable._fromList = fromList; + + // Pluck off n bytes from an array of buffers. + // Length is the combined lengths of all the buffers in the list. + // This function is designed to be inlinable, so please take care when making + // changes to the function body. + function fromList(n, state) { + // nothing buffered + if (state.length === 0) return null; + + var ret; + if (state.objectMode) ret = state.buffer.shift();else if (!n || n >= state.length) { + // read it all, truncate the list + if (state.decoder) ret = state.buffer.join('');else if (state.buffer.length === 1) ret = state.buffer.head.data;else ret = state.buffer.concat(state.length); + state.buffer.clear(); + } else { + // read part of list + ret = fromListPartial(n, state.buffer, state.decoder); + } + + return ret; + } + + // Extracts only enough buffered data to satisfy the amount requested. + // This function is designed to be inlinable, so please take care when making + // changes to the function body. + function fromListPartial(n, list, hasStrings) { + var ret; + if (n < list.head.data.length) { + // slice is the same for buffers and strings + ret = list.head.data.slice(0, n); + list.head.data = list.head.data.slice(n); + } else if (n === list.head.data.length) { + // first chunk is a perfect match + ret = list.shift(); + } else { + // result spans more than one buffer + ret = hasStrings ? copyFromBufferString(n, list) : copyFromBuffer(n, list); + } + return ret; + } + + // Copies a specified amount of characters from the list of buffered data + // chunks. + // This function is designed to be inlinable, so please take care when making + // changes to the function body. + function copyFromBufferString(n, list) { + var p = list.head; + var c = 1; + var ret = p.data; + n -= ret.length; + while (p = p.next) { + var str = p.data; + var nb = n > str.length ? str.length : n; + if (nb === str.length) ret += str;else ret += str.slice(0, n); + n -= nb; + if (n === 0) { + if (nb === str.length) { + ++c; + if (p.next) list.head = p.next;else list.head = list.tail = null; + } else { + list.head = p; + p.data = str.slice(nb); + } + break; + } + ++c; + } + list.length -= c; + return ret; + } + + // Copies a specified amount of bytes from the list of buffered data chunks. + // This function is designed to be inlinable, so please take care when making + // changes to the function body. + function copyFromBuffer(n, list) { + var ret = Buffer.allocUnsafe(n); + var p = list.head; + var c = 1; + p.data.copy(ret); + n -= p.data.length; + while (p = p.next) { + var buf = p.data; + var nb = n > buf.length ? buf.length : n; + buf.copy(ret, ret.length - n, 0, nb); + n -= nb; + if (n === 0) { + if (nb === buf.length) { + ++c; + if (p.next) list.head = p.next;else list.head = list.tail = null; + } else { + list.head = p; + p.data = buf.slice(nb); + } + break; + } + ++c; + } + list.length -= c; + return ret; + } + + function endReadable(stream) { + var state = stream._readableState; + + // If we get here before consuming all the bytes, then that is a + // bug in node. Should never happen. + if (state.length > 0) throw new Error('"endReadable()" called on non-empty stream'); + + if (!state.endEmitted) { + state.ended = true; + nextTick(endReadableNT, state, stream); + } + } + + function endReadableNT(state, stream) { + // Check that we didn't get one last unshift. + if (!state.endEmitted && state.length === 0) { + state.endEmitted = true; + stream.readable = false; + stream.emit('end'); + } + } + + function forEach(xs, f) { + for (var i = 0, l = xs.length; i < l; i++) { + f(xs[i], i); + } + } + + function indexOf(xs, x) { + for (var i = 0, l = xs.length; i < l; i++) { + if (xs[i] === x) return i; + } + return -1; + } + + // A bit simpler than readable streams. + Writable.WritableState = WritableState; + inherits$1(Writable, EventEmitter); + + function nop() {} + + function WriteReq(chunk, encoding, cb) { + this.chunk = chunk; + this.encoding = encoding; + this.callback = cb; + this.next = null; + } + + function WritableState(options, stream) { + Object.defineProperty(this, 'buffer', { + get: deprecate(function () { + return this.getBuffer(); + }, '_writableState.buffer is deprecated. Use _writableState.getBuffer ' + 'instead.') + }); + options = options || {}; + + // object stream flag to indicate whether or not this stream + // contains buffers or objects. + this.objectMode = !!options.objectMode; + + if (stream instanceof Duplex) this.objectMode = this.objectMode || !!options.writableObjectMode; + + // the point at which write() starts returning false + // Note: 0 is a valid value, means that we always return false if + // the entire buffer is not flushed immediately on write() + var hwm = options.highWaterMark; + var defaultHwm = this.objectMode ? 16 : 16 * 1024; + this.highWaterMark = hwm || hwm === 0 ? hwm : defaultHwm; + + // cast to ints. + this.highWaterMark = ~ ~this.highWaterMark; + + this.needDrain = false; + // at the start of calling end() + this.ending = false; + // when end() has been called, and returned + this.ended = false; + // when 'finish' is emitted + this.finished = false; + + // should we decode strings into buffers before passing to _write? + // this is here so that some node-core streams can optimize string + // handling at a lower level. + var noDecode = options.decodeStrings === false; + this.decodeStrings = !noDecode; + + // Crypto is kind of old and crusty. Historically, its default string + // encoding is 'binary' so we have to make this configurable. + // Everything else in the universe uses 'utf8', though. + this.defaultEncoding = options.defaultEncoding || 'utf8'; + + // not an actual buffer we keep track of, but a measurement + // of how much we're waiting to get pushed to some underlying + // socket or file. + this.length = 0; + + // a flag to see when we're in the middle of a write. + this.writing = false; + + // when true all writes will be buffered until .uncork() call + this.corked = 0; + + // a flag to be able to tell if the onwrite cb is called immediately, + // or on a later tick. We set this to true at first, because any + // actions that shouldn't happen until "later" should generally also + // not happen before the first write call. + this.sync = true; + + // a flag to know if we're processing previously buffered items, which + // may call the _write() callback in the same tick, so that we don't + // end up in an overlapped onwrite situation. + this.bufferProcessing = false; + + // the callback that's passed to _write(chunk,cb) + this.onwrite = function (er) { + onwrite(stream, er); + }; + + // the callback that the user supplies to write(chunk,encoding,cb) + this.writecb = null; + + // the amount that is being written when _write is called. + this.writelen = 0; + + this.bufferedRequest = null; + this.lastBufferedRequest = null; + + // number of pending user-supplied write callbacks + // this must be 0 before 'finish' can be emitted + this.pendingcb = 0; + + // emit prefinish if the only thing we're waiting for is _write cbs + // This is relevant for synchronous Transform streams + this.prefinished = false; + + // True if the error was already emitted and should not be thrown again + this.errorEmitted = false; + + // count buffered requests + this.bufferedRequestCount = 0; + + // allocate the first CorkedRequest, there is always + // one allocated and free to use, and we maintain at most two + this.corkedRequestsFree = new CorkedRequest(this); + } + + WritableState.prototype.getBuffer = function writableStateGetBuffer() { + var current = this.bufferedRequest; + var out = []; + while (current) { + out.push(current); + current = current.next; + } + return out; + }; + function Writable(options) { + + // Writable ctor is applied to Duplexes, though they're not + // instanceof Writable, they're instanceof Readable. + if (!(this instanceof Writable) && !(this instanceof Duplex)) return new Writable(options); + + this._writableState = new WritableState(options, this); + + // legacy. + this.writable = true; + + if (options) { + if (typeof options.write === 'function') this._write = options.write; + + if (typeof options.writev === 'function') this._writev = options.writev; + } + + EventEmitter.call(this); + } + + // Otherwise people can pipe Writable streams, which is just wrong. + Writable.prototype.pipe = function () { + this.emit('error', new Error('Cannot pipe, not readable')); + }; + + function writeAfterEnd(stream, cb) { + var er = new Error('write after end'); + // TODO: defer error events consistently everywhere, not just the cb + stream.emit('error', er); + nextTick(cb, er); + } + + // If we get something that is not a buffer, string, null, or undefined, + // and we're not in objectMode, then that's an error. + // Otherwise stream chunks are all considered to be of length=1, and the + // watermarks determine how many objects to keep in the buffer, rather than + // how many bytes or characters. + function validChunk(stream, state, chunk, cb) { + var valid = true; + var er = false; + // Always throw error if a null is written + // if we are not in object mode then throw + // if it is not a buffer, string, or undefined. + if (chunk === null) { + er = new TypeError('May not write null values to stream'); + } else if (!Buffer.isBuffer(chunk) && typeof chunk !== 'string' && chunk !== undefined && !state.objectMode) { + er = new TypeError('Invalid non-string/buffer chunk'); + } + if (er) { + stream.emit('error', er); + nextTick(cb, er); + valid = false; + } + return valid; + } + + Writable.prototype.write = function (chunk, encoding, cb) { + var state = this._writableState; + var ret = false; + + if (typeof encoding === 'function') { + cb = encoding; + encoding = null; + } + + if (Buffer.isBuffer(chunk)) encoding = 'buffer';else if (!encoding) encoding = state.defaultEncoding; + + if (typeof cb !== 'function') cb = nop; + + if (state.ended) writeAfterEnd(this, cb);else if (validChunk(this, state, chunk, cb)) { + state.pendingcb++; + ret = writeOrBuffer(this, state, chunk, encoding, cb); + } + + return ret; + }; + + Writable.prototype.cork = function () { + var state = this._writableState; + + state.corked++; + }; + + Writable.prototype.uncork = function () { + var state = this._writableState; + + if (state.corked) { + state.corked--; + + if (!state.writing && !state.corked && !state.finished && !state.bufferProcessing && state.bufferedRequest) clearBuffer(this, state); + } + }; + + Writable.prototype.setDefaultEncoding = function setDefaultEncoding(encoding) { + // node::ParseEncoding() requires lower case. + if (typeof encoding === 'string') encoding = encoding.toLowerCase(); + if (!(['hex', 'utf8', 'utf-8', 'ascii', 'binary', 'base64', 'ucs2', 'ucs-2', 'utf16le', 'utf-16le', 'raw'].indexOf((encoding + '').toLowerCase()) > -1)) throw new TypeError('Unknown encoding: ' + encoding); + this._writableState.defaultEncoding = encoding; + return this; + }; + + function decodeChunk(state, chunk, encoding) { + if (!state.objectMode && state.decodeStrings !== false && typeof chunk === 'string') { + chunk = Buffer.from(chunk, encoding); + } + return chunk; + } + + // if we're already writing something, then just put this + // in the queue, and wait our turn. Otherwise, call _write + // If we return false, then we need a drain event, so set that flag. + function writeOrBuffer(stream, state, chunk, encoding, cb) { + chunk = decodeChunk(state, chunk, encoding); + + if (Buffer.isBuffer(chunk)) encoding = 'buffer'; + var len = state.objectMode ? 1 : chunk.length; + + state.length += len; + + var ret = state.length < state.highWaterMark; + // we must ensure that previous needDrain will not be reset to false. + if (!ret) state.needDrain = true; + + if (state.writing || state.corked) { + var last = state.lastBufferedRequest; + state.lastBufferedRequest = new WriteReq(chunk, encoding, cb); + if (last) { + last.next = state.lastBufferedRequest; + } else { + state.bufferedRequest = state.lastBufferedRequest; + } + state.bufferedRequestCount += 1; + } else { + doWrite(stream, state, false, len, chunk, encoding, cb); + } + + return ret; + } + + function doWrite(stream, state, writev, len, chunk, encoding, cb) { + state.writelen = len; + state.writecb = cb; + state.writing = true; + state.sync = true; + if (writev) stream._writev(chunk, state.onwrite);else stream._write(chunk, encoding, state.onwrite); + state.sync = false; + } + + function onwriteError(stream, state, sync, er, cb) { + --state.pendingcb; + if (sync) nextTick(cb, er);else cb(er); + + stream._writableState.errorEmitted = true; + stream.emit('error', er); + } + + function onwriteStateUpdate(state) { + state.writing = false; + state.writecb = null; + state.length -= state.writelen; + state.writelen = 0; + } + + function onwrite(stream, er) { + var state = stream._writableState; + var sync = state.sync; + var cb = state.writecb; + + onwriteStateUpdate(state); + + if (er) onwriteError(stream, state, sync, er, cb);else { + // Check if we're actually ready to finish, but don't emit yet + var finished = needFinish(state); + + if (!finished && !state.corked && !state.bufferProcessing && state.bufferedRequest) { + clearBuffer(stream, state); + } + + if (sync) { + /**/ + nextTick(afterWrite, stream, state, finished, cb); + /**/ + } else { + afterWrite(stream, state, finished, cb); + } + } + } + + function afterWrite(stream, state, finished, cb) { + if (!finished) onwriteDrain(stream, state); + state.pendingcb--; + cb(); + finishMaybe(stream, state); + } + + // Must force callback to be called on nextTick, so that we don't + // emit 'drain' before the write() consumer gets the 'false' return + // value, and has a chance to attach a 'drain' listener. + function onwriteDrain(stream, state) { + if (state.length === 0 && state.needDrain) { + state.needDrain = false; + stream.emit('drain'); + } + } + + // if there's something in the buffer waiting, then process it + function clearBuffer(stream, state) { + state.bufferProcessing = true; + var entry = state.bufferedRequest; + + if (stream._writev && entry && entry.next) { + // Fast case, write everything using _writev() + var l = state.bufferedRequestCount; + var buffer = new Array(l); + var holder = state.corkedRequestsFree; + holder.entry = entry; + + var count = 0; + while (entry) { + buffer[count] = entry; + entry = entry.next; + count += 1; + } + + doWrite(stream, state, true, state.length, buffer, '', holder.finish); + + // doWrite is almost always async, defer these to save a bit of time + // as the hot path ends with doWrite + state.pendingcb++; + state.lastBufferedRequest = null; + if (holder.next) { + state.corkedRequestsFree = holder.next; + holder.next = null; + } else { + state.corkedRequestsFree = new CorkedRequest(state); + } + } else { + // Slow case, write chunks one-by-one + while (entry) { + var chunk = entry.chunk; + var encoding = entry.encoding; + var cb = entry.callback; + var len = state.objectMode ? 1 : chunk.length; + + doWrite(stream, state, false, len, chunk, encoding, cb); + entry = entry.next; + // if we didn't call the onwrite immediately, then + // it means that we need to wait until it does. + // also, that means that the chunk and cb are currently + // being processed, so move the buffer counter past them. + if (state.writing) { + break; + } + } + + if (entry === null) state.lastBufferedRequest = null; + } + + state.bufferedRequestCount = 0; + state.bufferedRequest = entry; + state.bufferProcessing = false; + } + + Writable.prototype._write = function (chunk, encoding, cb) { + cb(new Error('not implemented')); + }; + + Writable.prototype._writev = null; + + Writable.prototype.end = function (chunk, encoding, cb) { + var state = this._writableState; + + if (typeof chunk === 'function') { + cb = chunk; + chunk = null; + encoding = null; + } else if (typeof encoding === 'function') { + cb = encoding; + encoding = null; + } + + if (chunk !== null && chunk !== undefined) this.write(chunk, encoding); + + // .end() fully uncorks + if (state.corked) { + state.corked = 1; + this.uncork(); + } + + // ignore unnecessary end() calls. + if (!state.ending && !state.finished) endWritable(this, state, cb); + }; + + function needFinish(state) { + return state.ending && state.length === 0 && state.bufferedRequest === null && !state.finished && !state.writing; + } + + function prefinish(stream, state) { + if (!state.prefinished) { + state.prefinished = true; + stream.emit('prefinish'); + } + } + + function finishMaybe(stream, state) { + var need = needFinish(state); + if (need) { + if (state.pendingcb === 0) { + prefinish(stream, state); + state.finished = true; + stream.emit('finish'); + } else { + prefinish(stream, state); + } + } + return need; + } + + function endWritable(stream, state, cb) { + state.ending = true; + finishMaybe(stream, state); + if (cb) { + if (state.finished) nextTick(cb);else stream.once('finish', cb); + } + state.ended = true; + stream.writable = false; + } + + // It seems a linked list but it is not + // there will be only 2 of these for each stream + function CorkedRequest(state) { + var _this = this; + + this.next = null; + this.entry = null; + + this.finish = function (err) { + var entry = _this.entry; + _this.entry = null; + while (entry) { + var cb = entry.callback; + state.pendingcb--; + cb(err); + entry = entry.next; + } + if (state.corkedRequestsFree) { + state.corkedRequestsFree.next = _this; + } else { + state.corkedRequestsFree = _this; + } + }; + } + + inherits$1(Duplex, Readable); + + var keys = Object.keys(Writable.prototype); + for (var v = 0; v < keys.length; v++) { + var method = keys[v]; + if (!Duplex.prototype[method]) Duplex.prototype[method] = Writable.prototype[method]; + } + function Duplex(options) { + if (!(this instanceof Duplex)) return new Duplex(options); + + Readable.call(this, options); + Writable.call(this, options); + + if (options && options.readable === false) this.readable = false; + + if (options && options.writable === false) this.writable = false; + + this.allowHalfOpen = true; + if (options && options.allowHalfOpen === false) this.allowHalfOpen = false; + + this.once('end', onend); + } + + // the no-half-open enforcer + function onend() { + // if we allow half-open state, or if the writable side ended, + // then we're ok. + if (this.allowHalfOpen || this._writableState.ended) return; + + // no more data can be written. + // But allow more writes to happen in this tick. + nextTick(onEndNT, this); + } + + function onEndNT(self) { + self.end(); + } + + // a transform stream is a readable/writable stream where you do + inherits$1(Transform, Duplex); + + function TransformState(stream) { + this.afterTransform = function (er, data) { + return afterTransform(stream, er, data); + }; + + this.needTransform = false; + this.transforming = false; + this.writecb = null; + this.writechunk = null; + this.writeencoding = null; + } + + function afterTransform(stream, er, data) { + var ts = stream._transformState; + ts.transforming = false; + + var cb = ts.writecb; + + if (!cb) return stream.emit('error', new Error('no writecb in Transform class')); + + ts.writechunk = null; + ts.writecb = null; + + if (data !== null && data !== undefined) stream.push(data); + + cb(er); + + var rs = stream._readableState; + rs.reading = false; + if (rs.needReadable || rs.length < rs.highWaterMark) { + stream._read(rs.highWaterMark); + } + } + function Transform(options) { + if (!(this instanceof Transform)) return new Transform(options); + + Duplex.call(this, options); + + this._transformState = new TransformState(this); + + // when the writable side finishes, then flush out anything remaining. + var stream = this; + + // start out asking for a readable event once data is transformed. + this._readableState.needReadable = true; + + // we have implemented the _read method, and done the other things + // that Readable wants before the first _read call, so unset the + // sync guard flag. + this._readableState.sync = false; + + if (options) { + if (typeof options.transform === 'function') this._transform = options.transform; + + if (typeof options.flush === 'function') this._flush = options.flush; + } + + this.once('prefinish', function () { + if (typeof this._flush === 'function') this._flush(function (er) { + done(stream, er); + });else done(stream); + }); + } + + Transform.prototype.push = function (chunk, encoding) { + this._transformState.needTransform = false; + return Duplex.prototype.push.call(this, chunk, encoding); + }; + + // This is the part where you do stuff! + // override this function in implementation classes. + // 'chunk' is an input chunk. + // + // Call `push(newChunk)` to pass along transformed output + // to the readable side. You may call 'push' zero or more times. + // + // Call `cb(err)` when you are done with this chunk. If you pass + // an error, then that'll put the hurt on the whole operation. If you + // never call cb(), then you'll never get another chunk. + Transform.prototype._transform = function (chunk, encoding, cb) { + throw new Error('Not implemented'); + }; + + Transform.prototype._write = function (chunk, encoding, cb) { + var ts = this._transformState; + ts.writecb = cb; + ts.writechunk = chunk; + ts.writeencoding = encoding; + if (!ts.transforming) { + var rs = this._readableState; + if (ts.needTransform || rs.needReadable || rs.length < rs.highWaterMark) this._read(rs.highWaterMark); + } + }; + + // Doesn't matter what the args are here. + // _transform does all the work. + // That we got here means that the readable side wants more data. + Transform.prototype._read = function (n) { + var ts = this._transformState; + + if (ts.writechunk !== null && ts.writecb && !ts.transforming) { + ts.transforming = true; + this._transform(ts.writechunk, ts.writeencoding, ts.afterTransform); + } else { + // mark that we need a transform, so that any data that comes in + // will get processed, now that we've asked for it. + ts.needTransform = true; + } + }; + + function done(stream, er) { + if (er) return stream.emit('error', er); + + // if there's nothing in the write buffer, then that means + // that nothing more will ever be provided + var ws = stream._writableState; + var ts = stream._transformState; + + if (ws.length) throw new Error('Calling transform done when ws.length != 0'); + + if (ts.transforming) throw new Error('Calling transform done when still transforming'); + + return stream.push(null); + } + + inherits$1(PassThrough, Transform); + function PassThrough(options) { + if (!(this instanceof PassThrough)) return new PassThrough(options); + + Transform.call(this, options); + } + + PassThrough.prototype._transform = function (chunk, encoding, cb) { + cb(null, chunk); + }; + + inherits$1(Stream, EventEmitter); + Stream.Readable = Readable; + Stream.Writable = Writable; + Stream.Duplex = Duplex; + Stream.Transform = Transform; + Stream.PassThrough = PassThrough; + + // Backwards-compat with node 0.4.x + Stream.Stream = Stream; + + // old-style streams. Note that the pipe method (the only relevant + // part of this class) is overridden in the Readable class. + + function Stream() { + EventEmitter.call(this); + } + + Stream.prototype.pipe = function(dest, options) { + var source = this; + + function ondata(chunk) { + if (dest.writable) { + if (false === dest.write(chunk) && source.pause) { + source.pause(); + } + } + } + + source.on('data', ondata); + + function ondrain() { + if (source.readable && source.resume) { + source.resume(); + } + } + + dest.on('drain', ondrain); + + // If the 'end' option is not supplied, dest.end() will be called when + // source gets the 'end' or 'close' events. Only dest.end() once. + if (!dest._isStdio && (!options || options.end !== false)) { + source.on('end', onend); + source.on('close', onclose); + } + + var didOnEnd = false; + function onend() { + if (didOnEnd) return; + didOnEnd = true; + + dest.end(); + } + + + function onclose() { + if (didOnEnd) return; + didOnEnd = true; + + if (typeof dest.destroy === 'function') dest.destroy(); + } + + // don't leave dangling pipes when there are errors. + function onerror(er) { + cleanup(); + if (EventEmitter.listenerCount(this, 'error') === 0) { + throw er; // Unhandled stream error in pipe. + } + } + + source.on('error', onerror); + dest.on('error', onerror); + + // remove all the event listeners that were added. + function cleanup() { + source.removeListener('data', ondata); + dest.removeListener('drain', ondrain); + + source.removeListener('end', onend); + source.removeListener('close', onclose); + + source.removeListener('error', onerror); + dest.removeListener('error', onerror); + + source.removeListener('end', cleanup); + source.removeListener('close', cleanup); + + dest.removeListener('close', cleanup); + } + + source.on('end', cleanup); + source.on('close', cleanup); + + dest.on('close', cleanup); + + dest.emit('pipe', source); + + // Allow for unix-like usage: A.pipe(B).pipe(C) + return dest; + }; + + class CsvError extends Error { + constructor(code, message, ...contexts) { + if(Array.isArray(message)) message = message.join(' '); + super(message); + if(Error.captureStackTrace !== undefined){ + Error.captureStackTrace(this, CsvError); + } + this.code = code; + for(const context of contexts){ + for(const key in context){ + const value = context[key]; + this[key] = isBuffer(value) ? value.toString() : value == null ? value : JSON.parse(JSON.stringify(value)); + } + } + } + } + + const is_object = function(obj){ + return typeof obj === 'object' && obj !== null && ! Array.isArray(obj); + }; + + // Lodash implementation of `get` + + const charCodeOfDot = '.'.charCodeAt(0); + const reEscapeChar = /\\(\\)?/g; + const rePropName = RegExp( + // Match anything that isn't a dot or bracket. + '[^.[\\]]+' + '|' + + // Or match property names within brackets. + '\\[(?:' + + // Match a non-string expression. + '([^"\'][^[]*)' + '|' + + // Or match strings (supports escaping characters). + '(["\'])((?:(?!\\2)[^\\\\]|\\\\.)*?)\\2' + + ')\\]'+ '|' + + // Or match "" as the space between consecutive dots or empty brackets. + '(?=(?:\\.|\\[\\])(?:\\.|\\[\\]|$))' + , 'g'); + const reIsDeepProp = /\.|\[(?:[^[\]]*|(["'])(?:(?!\1)[^\\]|\\.)*?\1)\]/; + const reIsPlainProp = /^\w*$/; + const getTag = function(value){ + return Object.prototype.toString.call(value); + }; + const isSymbol = function(value){ + const type = typeof value; + return type === 'symbol' || (type === 'object' && value && getTag(value) === '[object Symbol]'); + }; + const isKey = function(value, object){ + if(Array.isArray(value)){ + return false; + } + const type = typeof value; + if(type === 'number' || type === 'symbol' || type === 'boolean' || !value || isSymbol(value)){ + return true; + } + return reIsPlainProp.test(value) || !reIsDeepProp.test(value) || + (object != null && value in Object(object)); + }; + const stringToPath = function(string){ + const result = []; + if(string.charCodeAt(0) === charCodeOfDot){ + result.push(''); + } + string.replace(rePropName, function(match, expression, quote, subString){ + let key = match; + if(quote){ + key = subString.replace(reEscapeChar, '$1'); + }else if(expression){ + key = expression.trim(); + } + result.push(key); + }); + return result; + }; + const castPath = function(value, object){ + if(Array.isArray(value)){ + return value; + } else { + return isKey(value, object) ? [value] : stringToPath(value); + } + }; + const toKey = function(value){ + if(typeof value === 'string' || isSymbol(value)) + return value; + const result = `${value}`; + // eslint-disable-next-line + return (result == '0' && (1 / value) == -INFINITY) ? '-0' : result; + }; + const get = function(object, path){ + path = castPath(path, object); + let index = 0; + const length = path.length; + while(object != null && index < length){ + object = object[toKey(path[index++])]; + } + return (index && index === length) ? object : undefined; + }; + + const normalize_columns = function(columns){ + if(columns === undefined || columns === null){ + return [undefined, undefined]; + } + if(typeof columns !== 'object'){ + return [Error('Invalid option "columns": expect an array or an object')]; + } + if(!Array.isArray(columns)){ + const newcolumns = []; + for(const k in columns){ + newcolumns.push({ + key: k, + header: columns[k] + }); + } + columns = newcolumns; + }else { + const newcolumns = []; + for(const column of columns){ + if(typeof column === 'string'){ + newcolumns.push({ + key: column, + header: column + }); + }else if(typeof column === 'object' && column !== null && !Array.isArray(column)){ + if(!column.key){ + return [Error('Invalid column definition: property "key" is required')]; + } + if(column.header === undefined){ + column.header = column.key; + } + newcolumns.push(column); + }else { + return [Error('Invalid column definition: expect a string or an object')]; + } + } + columns = newcolumns; + } + return [undefined, columns]; + }; + + const underscore = function(str){ + return str.replace(/([A-Z])/g, function(_, match){ + return '_' + match.toLowerCase(); + }); + }; + + const normalize_options = function(opts) { + const options = {}; + // Merge with user options + for(const opt in opts){ + options[underscore(opt)] = opts[opt]; + } + // Normalize option `bom` + if(options.bom === undefined || options.bom === null || options.bom === false){ + options.bom = false; + }else if(options.bom !== true){ + return [new CsvError('CSV_OPTION_BOOLEAN_INVALID_TYPE', [ + 'option `bom` is optional and must be a boolean value,', + `got ${JSON.stringify(options.bom)}` + ])]; + } + // Normalize option `delimiter` + if(options.delimiter === undefined || options.delimiter === null){ + options.delimiter = ','; + }else if(isBuffer(options.delimiter)){ + options.delimiter = options.delimiter.toString(); + }else if(typeof options.delimiter !== 'string'){ + return [new CsvError('CSV_OPTION_DELIMITER_INVALID_TYPE', [ + 'option `delimiter` must be a buffer or a string,', + `got ${JSON.stringify(options.delimiter)}` + ])]; + } + // Normalize option `quote` + if(options.quote === undefined || options.quote === null){ + options.quote = '"'; + }else if(options.quote === true){ + options.quote = '"'; + }else if(options.quote === false){ + options.quote = ''; + }else if (isBuffer(options.quote)){ + options.quote = options.quote.toString(); + }else if(typeof options.quote !== 'string'){ + return [new CsvError('CSV_OPTION_QUOTE_INVALID_TYPE', [ + 'option `quote` must be a boolean, a buffer or a string,', + `got ${JSON.stringify(options.quote)}` + ])]; + } + // Normalize option `quoted` + if(options.quoted === undefined || options.quoted === null){ + options.quoted = false; + } + // Normalize option `quoted_empty` + if(options.quoted_empty === undefined || options.quoted_empty === null){ + options.quoted_empty = undefined; + } + // Normalize option `quoted_match` + if(options.quoted_match === undefined || options.quoted_match === null || options.quoted_match === false){ + options.quoted_match = null; + }else if(!Array.isArray(options.quoted_match)){ + options.quoted_match = [options.quoted_match]; + } + if(options.quoted_match){ + for(const quoted_match of options.quoted_match){ + const isString = typeof quoted_match === 'string'; + const isRegExp = quoted_match instanceof RegExp; + if(!isString && !isRegExp){ + return [Error(`Invalid Option: quoted_match must be a string or a regex, got ${JSON.stringify(quoted_match)}`)]; + } + } + } + // Normalize option `quoted_string` + if(options.quoted_string === undefined || options.quoted_string === null){ + options.quoted_string = false; + } + // Normalize option `eof` + if(options.eof === undefined || options.eof === null){ + options.eof = true; + } + // Normalize option `escape` + if(options.escape === undefined || options.escape === null){ + options.escape = '"'; + }else if(isBuffer(options.escape)){ + options.escape = options.escape.toString(); + }else if(typeof options.escape !== 'string'){ + return [Error(`Invalid Option: escape must be a buffer or a string, got ${JSON.stringify(options.escape)}`)]; + } + if (options.escape.length > 1){ + return [Error(`Invalid Option: escape must be one character, got ${options.escape.length} characters`)]; + } + // Normalize option `header` + if(options.header === undefined || options.header === null){ + options.header = false; + } + // Normalize option `columns` + const [errColumns, columns] = normalize_columns(options.columns); + if(errColumns !== undefined) return [errColumns]; + options.columns = columns; + // Normalize option `quoted` + if(options.quoted === undefined || options.quoted === null){ + options.quoted = false; + } + // Normalize option `cast` + if(options.cast === undefined || options.cast === null){ + options.cast = {}; + } + // Normalize option cast.bigint + if(options.cast.bigint === undefined || options.cast.bigint === null){ + // Cast boolean to string by default + options.cast.bigint = value => '' + value; + } + // Normalize option cast.boolean + if(options.cast.boolean === undefined || options.cast.boolean === null){ + // Cast boolean to string by default + options.cast.boolean = value => value ? '1' : ''; + } + // Normalize option cast.date + if(options.cast.date === undefined || options.cast.date === null){ + // Cast date to timestamp string by default + options.cast.date = value => '' + value.getTime(); + } + // Normalize option cast.number + if(options.cast.number === undefined || options.cast.number === null){ + // Cast number to string using native casting by default + options.cast.number = value => '' + value; + } + // Normalize option cast.object + if(options.cast.object === undefined || options.cast.object === null){ + // Stringify object as JSON by default + options.cast.object = value => JSON.stringify(value); + } + // Normalize option cast.string + if(options.cast.string === undefined || options.cast.string === null){ + // Leave string untouched + options.cast.string = function(value){return value;}; + } + // Normalize option `on_record` + if(options.on_record !== undefined && typeof options.on_record !== 'function'){ + return [Error(`Invalid Option: "on_record" must be a function.`)]; + } + // Normalize option `record_delimiter` + if(options.record_delimiter === undefined || options.record_delimiter === null){ + options.record_delimiter = '\n'; + }else if(isBuffer(options.record_delimiter)){ + options.record_delimiter = options.record_delimiter.toString(); + }else if(typeof options.record_delimiter !== 'string'){ + return [Error(`Invalid Option: record_delimiter must be a buffer or a string, got ${JSON.stringify(options.record_delimiter)}`)]; + } + switch(options.record_delimiter){ + case 'auto': + options.record_delimiter = null; + break; + case 'unix': + options.record_delimiter = "\n"; + break; + case 'mac': + options.record_delimiter = "\r"; + break; + case 'windows': + options.record_delimiter = "\r\n"; + break; + case 'ascii': + options.record_delimiter = "\u001e"; + break; + case 'unicode': + options.record_delimiter = "\u2028"; + break; + } + return [undefined, options]; + }; + + const bom_utf8 = Buffer.from([239, 187, 191]); + + const stringifier = function(options, state, info){ + return { + options: options, + state: state, + info: info, + __transform: function(chunk, push){ + // Chunk validation + if(!Array.isArray(chunk) && typeof chunk !== 'object'){ + return Error(`Invalid Record: expect an array or an object, got ${JSON.stringify(chunk)}`); + } + // Detect columns from the first record + if(this.info.records === 0){ + if(Array.isArray(chunk)){ + if(this.options.header === true && this.options.columns === undefined){ + return Error('Undiscoverable Columns: header option requires column option or object records'); + } + }else if(this.options.columns === undefined){ + const [err, columns] = normalize_columns(Object.keys(chunk)); + if(err) return; + this.options.columns = columns; + } + } + // Emit the header + if(this.info.records === 0){ + this.bom(push); + const err = this.headers(push); + if(err) return err; + } + // Emit and stringify the record if an object or an array + try{ + // this.emit('record', chunk, this.info.records); + if(this.options.on_record){ + this.options.on_record(chunk, this.info.records); + } + }catch(err){ + return err; + } + // Convert the record into a string + let err, chunk_string; + if(this.options.eof){ + [err, chunk_string] = this.stringify(chunk); + if(err) return err; + if(chunk_string === undefined){ + return; + }else { + chunk_string = chunk_string + this.options.record_delimiter; + } + }else { + [err, chunk_string] = this.stringify(chunk); + if(err) return err; + if(chunk_string === undefined){ + return; + }else { + if(this.options.header || this.info.records){ + chunk_string = this.options.record_delimiter + chunk_string; + } + } + } + // Emit the csv + this.info.records++; + push(chunk_string); + }, + stringify: function(chunk, chunkIsHeader=false){ + if(typeof chunk !== 'object'){ + return [undefined, chunk]; + } + const {columns} = this.options; + const record = []; + // Record is an array + if(Array.isArray(chunk)){ + // We are getting an array but the user has specified output columns. In + // this case, we respect the columns indexes + if(columns){ + chunk.splice(columns.length); + } + // Cast record elements + for(let i=0; i= 0; + const containsQuote = (quote !== '') && value.indexOf(quote) >= 0; + const containsEscape = value.indexOf(escape) >= 0 && (escape !== quote); + const containsRecordDelimiter = value.indexOf(record_delimiter) >= 0; + const quotedString = quoted_string && typeof field === 'string'; + let quotedMatch = quoted_match && quoted_match.filter(quoted_match => { + if(typeof quoted_match === 'string'){ + return value.indexOf(quoted_match) !== -1; + }else { + return quoted_match.test(value); + } + }); + quotedMatch = quotedMatch && quotedMatch.length > 0; + const shouldQuote = containsQuote === true || containsdelimiter || containsRecordDelimiter || quoted || quotedString || quotedMatch; + if(shouldQuote === true && containsEscape === true){ + const regexp = escape === '\\' + ? new RegExp(escape + escape, 'g') + : new RegExp(escape, 'g'); + value = value.replace(regexp, escape + escape); + } + if(containsQuote === true){ + const regexp = new RegExp(quote,'g'); + value = value.replace(regexp, escape + quote); + } + if(shouldQuote === true){ + value = quote + value + quote; + } + csvrecord += value; + }else if(quoted_empty === true || (field === '' && quoted_string === true && quoted_empty !== false)){ + csvrecord += quote + quote; + } + if(i !== record.length - 1){ + csvrecord += delimiter; + } + } + return [undefined, csvrecord]; + }, + bom: function(push){ + if(this.options.bom !== true){ + return; + } + push(bom_utf8); + }, + headers: function(push){ + if(this.options.header === false){ + return; + } + if(this.options.columns === undefined){ + return; + } + let err; + let headers = this.options.columns.map(column => column.header); + if(this.options.eof){ + [err, headers] = this.stringify(headers, true); + headers += this.options.record_delimiter; + }else { + [err, headers] = this.stringify(headers); + } + if(err) return err; + push(headers); + }, + __cast: function(value, context){ + const type = typeof value; + try{ + if(type === 'string'){ // Fine for 99% of the cases + return [undefined, this.options.cast.string(value, context)]; + }else if(type === 'bigint'){ + return [undefined, this.options.cast.bigint(value, context)]; + }else if(type === 'number'){ + return [undefined, this.options.cast.number(value, context)]; + }else if(type === 'boolean'){ + return [undefined, this.options.cast.boolean(value, context)]; + }else if(value instanceof Date){ + return [undefined, this.options.cast.date(value, context)]; + }else if(type === 'object' && value !== null){ + return [undefined, this.options.cast.object(value, context)]; + }else { + return [undefined, value, value]; + } + }catch(err){ + return [err]; + } + } + }; + }; + + class Stringifier extends Transform { + constructor(opts = {}){ + super({...{writableObjectMode: true}, ...opts}); + const [err, options] = normalize_options(opts); + if(err !== undefined) throw err; + // Expose options + this.options = options; + // Internal state + this.state = { + stop: false + }; + // Information + this.info = { + records: 0 + }; + this.api = stringifier(this.options, this.state, this.info); + this.api.options.on_record = (...args) => { + this.emit('record', ...args); + }; + } + _transform(chunk, encoding, callback){ + if(this.state.stop === true){ + return; + } + const err = this.api.__transform(chunk, this.push.bind(this)); + if(err !== undefined){ + this.state.stop = true; + } + callback(err); + } + _flush(callback){ + if(this.state.stop === true){ + // Note, Node.js 12 call flush even after an error, we must prevent + // `callback` from being called in flush without any error. + return; + } + if(this.info.records === 0){ + this.api.bom(this.push.bind(this)); + const err = this.api.headers(this.push.bind(this)); + if(err) callback(err); + } + callback(); + } + } + + const stringify = function(){ + let data, options, callback; + for(const i in arguments){ + const argument = arguments[i]; + const type = typeof argument; + if(data === undefined && (Array.isArray(argument))){ + data = argument; + }else if(options === undefined && is_object(argument)){ + options = argument; + }else if(callback === undefined && type === 'function'){ + callback = argument; + }else { + throw new CsvError('CSV_INVALID_ARGUMENT', [ + 'Invalid argument:', + `got ${JSON.stringify(argument)} at index ${i}` + ]); + } + } + const stringifier = new Stringifier(options); + if(callback){ + const chunks = []; + stringifier.on('readable', function(){ + let chunk; + while((chunk = this.read()) !== null){ + chunks.push(chunk); + } + }); + stringifier.on('error', function(err){ + callback(err); + }); + stringifier.on('end', function(){ + callback(undefined, chunks.join('')); + }); + } + if(data !== undefined){ + const writer = function(){ + for(const record of data){ + stringifier.write(record); + } + stringifier.end(); + }; + // Support Deno, Rollup doesnt provide a shim for setImmediate + if(typeof setImmediate === 'function'){ + setImmediate(writer); + }else { + setTimeout(writer, 0); + } + } + return stringifier; + }; + + exports.CsvError = CsvError; + exports.Stringifier = Stringifier; + exports.stringify = stringify; + + Object.defineProperty(exports, '__esModule', { value: true }); })); diff --git a/packages/csv-stringify/dist/umd/sync.js b/packages/csv-stringify/dist/umd/sync.js index 33c7680d9..1db0ff077 100644 --- a/packages/csv-stringify/dist/umd/sync.js +++ b/packages/csv-stringify/dist/umd/sync.js @@ -205,7 +205,7 @@ var toString = {}.toString; - var isArray$1 = Array.isArray || function (arr) { + var isArray = Array.isArray || function (arr) { return toString.call(arr) == '[object Array]'; }; @@ -473,7 +473,7 @@ return fromArrayLike(that, obj) } - if (obj.type === 'Buffer' && isArray$1(obj.data)) { + if (obj.type === 'Buffer' && isArray(obj.data)) { return fromArrayLike(that, obj.data) } } @@ -538,7 +538,7 @@ }; Buffer.concat = function concat (list, length) { - if (!isArray$1(list)) { + if (!isArray(list)) { throw new TypeError('"list" argument must be an Array of Buffers') } @@ -1969,3038 +1969,6 @@ return typeof obj.readFloatLE === 'function' && typeof obj.slice === 'function' && isFastBuffer(obj.slice(0, 0)) } - var domain; - - // This constructor is used to store event handlers. Instantiating this is - // faster than explicitly calling `Object.create(null)` to get a "clean" empty - // object (tested with v8 v4.9). - function EventHandlers() {} - EventHandlers.prototype = Object.create(null); - - function EventEmitter() { - EventEmitter.init.call(this); - } - - // nodejs oddity - // require('events') === require('events').EventEmitter - EventEmitter.EventEmitter = EventEmitter; - - EventEmitter.usingDomains = false; - - EventEmitter.prototype.domain = undefined; - EventEmitter.prototype._events = undefined; - EventEmitter.prototype._maxListeners = undefined; - - // By default EventEmitters will print a warning if more than 10 listeners are - // added to it. This is a useful default which helps finding memory leaks. - EventEmitter.defaultMaxListeners = 10; - - EventEmitter.init = function() { - this.domain = null; - if (EventEmitter.usingDomains) { - // if there is an active domain, then attach to it. - if (domain.active ) ; - } - - if (!this._events || this._events === Object.getPrototypeOf(this)._events) { - this._events = new EventHandlers(); - this._eventsCount = 0; - } - - this._maxListeners = this._maxListeners || undefined; - }; - - // Obviously not all Emitters should be limited to 10. This function allows - // that to be increased. Set to zero for unlimited. - EventEmitter.prototype.setMaxListeners = function setMaxListeners(n) { - if (typeof n !== 'number' || n < 0 || isNaN(n)) - throw new TypeError('"n" argument must be a positive number'); - this._maxListeners = n; - return this; - }; - - function $getMaxListeners(that) { - if (that._maxListeners === undefined) - return EventEmitter.defaultMaxListeners; - return that._maxListeners; - } - - EventEmitter.prototype.getMaxListeners = function getMaxListeners() { - return $getMaxListeners(this); - }; - - // These standalone emit* functions are used to optimize calling of event - // handlers for fast cases because emit() itself often has a variable number of - // arguments and can be deoptimized because of that. These functions always have - // the same number of arguments and thus do not get deoptimized, so the code - // inside them can execute faster. - function emitNone(handler, isFn, self) { - if (isFn) - handler.call(self); - else { - var len = handler.length; - var listeners = arrayClone(handler, len); - for (var i = 0; i < len; ++i) - listeners[i].call(self); - } - } - function emitOne(handler, isFn, self, arg1) { - if (isFn) - handler.call(self, arg1); - else { - var len = handler.length; - var listeners = arrayClone(handler, len); - for (var i = 0; i < len; ++i) - listeners[i].call(self, arg1); - } - } - function emitTwo(handler, isFn, self, arg1, arg2) { - if (isFn) - handler.call(self, arg1, arg2); - else { - var len = handler.length; - var listeners = arrayClone(handler, len); - for (var i = 0; i < len; ++i) - listeners[i].call(self, arg1, arg2); - } - } - function emitThree(handler, isFn, self, arg1, arg2, arg3) { - if (isFn) - handler.call(self, arg1, arg2, arg3); - else { - var len = handler.length; - var listeners = arrayClone(handler, len); - for (var i = 0; i < len; ++i) - listeners[i].call(self, arg1, arg2, arg3); - } - } - - function emitMany(handler, isFn, self, args) { - if (isFn) - handler.apply(self, args); - else { - var len = handler.length; - var listeners = arrayClone(handler, len); - for (var i = 0; i < len; ++i) - listeners[i].apply(self, args); - } - } - - EventEmitter.prototype.emit = function emit(type) { - var er, handler, len, args, i, events, domain; - var doError = (type === 'error'); - - events = this._events; - if (events) - doError = (doError && events.error == null); - else if (!doError) - return false; - - domain = this.domain; - - // If there is no 'error' event listener then throw. - if (doError) { - er = arguments[1]; - if (domain) { - if (!er) - er = new Error('Uncaught, unspecified "error" event'); - er.domainEmitter = this; - er.domain = domain; - er.domainThrown = false; - domain.emit('error', er); - } else if (er instanceof Error) { - throw er; // Unhandled 'error' event - } else { - // At least give some kind of context to the user - var err = new Error('Uncaught, unspecified "error" event. (' + er + ')'); - err.context = er; - throw err; - } - return false; - } - - handler = events[type]; - - if (!handler) - return false; - - var isFn = typeof handler === 'function'; - len = arguments.length; - switch (len) { - // fast cases - case 1: - emitNone(handler, isFn, this); - break; - case 2: - emitOne(handler, isFn, this, arguments[1]); - break; - case 3: - emitTwo(handler, isFn, this, arguments[1], arguments[2]); - break; - case 4: - emitThree(handler, isFn, this, arguments[1], arguments[2], arguments[3]); - break; - // slower - default: - args = new Array(len - 1); - for (i = 1; i < len; i++) - args[i - 1] = arguments[i]; - emitMany(handler, isFn, this, args); - } - - return true; - }; - - function _addListener(target, type, listener, prepend) { - var m; - var events; - var existing; - - if (typeof listener !== 'function') - throw new TypeError('"listener" argument must be a function'); - - events = target._events; - if (!events) { - events = target._events = new EventHandlers(); - target._eventsCount = 0; - } else { - // To avoid recursion in the case that type === "newListener"! Before - // adding it to the listeners, first emit "newListener". - if (events.newListener) { - target.emit('newListener', type, - listener.listener ? listener.listener : listener); - - // Re-assign `events` because a newListener handler could have caused the - // this._events to be assigned to a new object - events = target._events; - } - existing = events[type]; - } - - if (!existing) { - // Optimize the case of one listener. Don't need the extra array object. - existing = events[type] = listener; - ++target._eventsCount; - } else { - if (typeof existing === 'function') { - // Adding the second element, need to change to array. - existing = events[type] = prepend ? [listener, existing] : - [existing, listener]; - } else { - // If we've already got an array, just append. - if (prepend) { - existing.unshift(listener); - } else { - existing.push(listener); - } - } - - // Check for listener leak - if (!existing.warned) { - m = $getMaxListeners(target); - if (m && m > 0 && existing.length > m) { - existing.warned = true; - var w = new Error('Possible EventEmitter memory leak detected. ' + - existing.length + ' ' + type + ' listeners added. ' + - 'Use emitter.setMaxListeners() to increase limit'); - w.name = 'MaxListenersExceededWarning'; - w.emitter = target; - w.type = type; - w.count = existing.length; - emitWarning(w); - } - } - } - - return target; - } - function emitWarning(e) { - typeof console.warn === 'function' ? console.warn(e) : console.log(e); - } - EventEmitter.prototype.addListener = function addListener(type, listener) { - return _addListener(this, type, listener, false); - }; - - EventEmitter.prototype.on = EventEmitter.prototype.addListener; - - EventEmitter.prototype.prependListener = - function prependListener(type, listener) { - return _addListener(this, type, listener, true); - }; - - function _onceWrap(target, type, listener) { - var fired = false; - function g() { - target.removeListener(type, g); - if (!fired) { - fired = true; - listener.apply(target, arguments); - } - } - g.listener = listener; - return g; - } - - EventEmitter.prototype.once = function once(type, listener) { - if (typeof listener !== 'function') - throw new TypeError('"listener" argument must be a function'); - this.on(type, _onceWrap(this, type, listener)); - return this; - }; - - EventEmitter.prototype.prependOnceListener = - function prependOnceListener(type, listener) { - if (typeof listener !== 'function') - throw new TypeError('"listener" argument must be a function'); - this.prependListener(type, _onceWrap(this, type, listener)); - return this; - }; - - // emits a 'removeListener' event iff the listener was removed - EventEmitter.prototype.removeListener = - function removeListener(type, listener) { - var list, events, position, i, originalListener; - - if (typeof listener !== 'function') - throw new TypeError('"listener" argument must be a function'); - - events = this._events; - if (!events) - return this; - - list = events[type]; - if (!list) - return this; - - if (list === listener || (list.listener && list.listener === listener)) { - if (--this._eventsCount === 0) - this._events = new EventHandlers(); - else { - delete events[type]; - if (events.removeListener) - this.emit('removeListener', type, list.listener || listener); - } - } else if (typeof list !== 'function') { - position = -1; - - for (i = list.length; i-- > 0;) { - if (list[i] === listener || - (list[i].listener && list[i].listener === listener)) { - originalListener = list[i].listener; - position = i; - break; - } - } - - if (position < 0) - return this; - - if (list.length === 1) { - list[0] = undefined; - if (--this._eventsCount === 0) { - this._events = new EventHandlers(); - return this; - } else { - delete events[type]; - } - } else { - spliceOne(list, position); - } - - if (events.removeListener) - this.emit('removeListener', type, originalListener || listener); - } - - return this; - }; - - EventEmitter.prototype.removeAllListeners = - function removeAllListeners(type) { - var listeners, events; - - events = this._events; - if (!events) - return this; - - // not listening for removeListener, no need to emit - if (!events.removeListener) { - if (arguments.length === 0) { - this._events = new EventHandlers(); - this._eventsCount = 0; - } else if (events[type]) { - if (--this._eventsCount === 0) - this._events = new EventHandlers(); - else - delete events[type]; - } - return this; - } - - // emit removeListener for all listeners on all events - if (arguments.length === 0) { - var keys = Object.keys(events); - for (var i = 0, key; i < keys.length; ++i) { - key = keys[i]; - if (key === 'removeListener') continue; - this.removeAllListeners(key); - } - this.removeAllListeners('removeListener'); - this._events = new EventHandlers(); - this._eventsCount = 0; - return this; - } - - listeners = events[type]; - - if (typeof listeners === 'function') { - this.removeListener(type, listeners); - } else if (listeners) { - // LIFO order - do { - this.removeListener(type, listeners[listeners.length - 1]); - } while (listeners[0]); - } - - return this; - }; - - EventEmitter.prototype.listeners = function listeners(type) { - var evlistener; - var ret; - var events = this._events; - - if (!events) - ret = []; - else { - evlistener = events[type]; - if (!evlistener) - ret = []; - else if (typeof evlistener === 'function') - ret = [evlistener.listener || evlistener]; - else - ret = unwrapListeners(evlistener); - } - - return ret; - }; - - EventEmitter.listenerCount = function(emitter, type) { - if (typeof emitter.listenerCount === 'function') { - return emitter.listenerCount(type); - } else { - return listenerCount$1.call(emitter, type); - } - }; - - EventEmitter.prototype.listenerCount = listenerCount$1; - function listenerCount$1(type) { - var events = this._events; - - if (events) { - var evlistener = events[type]; - - if (typeof evlistener === 'function') { - return 1; - } else if (evlistener) { - return evlistener.length; - } - } - - return 0; - } - - EventEmitter.prototype.eventNames = function eventNames() { - return this._eventsCount > 0 ? Reflect.ownKeys(this._events) : []; - }; - - // About 1.5x faster than the two-arg version of Array#splice(). - function spliceOne(list, index) { - for (var i = index, k = i + 1, n = list.length; k < n; i += 1, k += 1) - list[i] = list[k]; - list.pop(); - } - - function arrayClone(arr, i) { - var copy = new Array(i); - while (i--) - copy[i] = arr[i]; - return copy; - } - - function unwrapListeners(arr) { - var ret = new Array(arr.length); - for (var i = 0; i < ret.length; ++i) { - ret[i] = arr[i].listener || arr[i]; - } - return ret; - } - - // shim for using process in browser - // based off https://github.com/defunctzombie/node-process/blob/master/browser.js - - function defaultSetTimout() { - throw new Error('setTimeout has not been defined'); - } - function defaultClearTimeout () { - throw new Error('clearTimeout has not been defined'); - } - var cachedSetTimeout = defaultSetTimout; - var cachedClearTimeout = defaultClearTimeout; - if (typeof global$1.setTimeout === 'function') { - cachedSetTimeout = setTimeout; - } - if (typeof global$1.clearTimeout === 'function') { - cachedClearTimeout = clearTimeout; - } - - function runTimeout(fun) { - if (cachedSetTimeout === setTimeout) { - //normal enviroments in sane situations - return setTimeout(fun, 0); - } - // if setTimeout wasn't available but was latter defined - if ((cachedSetTimeout === defaultSetTimout || !cachedSetTimeout) && setTimeout) { - cachedSetTimeout = setTimeout; - return setTimeout(fun, 0); - } - try { - // when when somebody has screwed with setTimeout but no I.E. maddness - return cachedSetTimeout(fun, 0); - } catch(e){ - try { - // When we are in I.E. but the script has been evaled so I.E. doesn't trust the global object when called normally - return cachedSetTimeout.call(null, fun, 0); - } catch(e){ - // same as above but when it's a version of I.E. that must have the global object for 'this', hopfully our context correct otherwise it will throw a global error - return cachedSetTimeout.call(this, fun, 0); - } - } - - - } - function runClearTimeout(marker) { - if (cachedClearTimeout === clearTimeout) { - //normal enviroments in sane situations - return clearTimeout(marker); - } - // if clearTimeout wasn't available but was latter defined - if ((cachedClearTimeout === defaultClearTimeout || !cachedClearTimeout) && clearTimeout) { - cachedClearTimeout = clearTimeout; - return clearTimeout(marker); - } - try { - // when when somebody has screwed with setTimeout but no I.E. maddness - return cachedClearTimeout(marker); - } catch (e){ - try { - // When we are in I.E. but the script has been evaled so I.E. doesn't trust the global object when called normally - return cachedClearTimeout.call(null, marker); - } catch (e){ - // same as above but when it's a version of I.E. that must have the global object for 'this', hopfully our context correct otherwise it will throw a global error. - // Some versions of I.E. have different rules for clearTimeout vs setTimeout - return cachedClearTimeout.call(this, marker); - } - } - - - - } - var queue = []; - var draining = false; - var currentQueue; - var queueIndex = -1; - - function cleanUpNextTick() { - if (!draining || !currentQueue) { - return; - } - draining = false; - if (currentQueue.length) { - queue = currentQueue.concat(queue); - } else { - queueIndex = -1; - } - if (queue.length) { - drainQueue(); - } - } - - function drainQueue() { - if (draining) { - return; - } - var timeout = runTimeout(cleanUpNextTick); - draining = true; - - var len = queue.length; - while(len) { - currentQueue = queue; - queue = []; - while (++queueIndex < len) { - if (currentQueue) { - currentQueue[queueIndex].run(); - } - } - queueIndex = -1; - len = queue.length; - } - currentQueue = null; - draining = false; - runClearTimeout(timeout); - } - function nextTick(fun) { - var args = new Array(arguments.length - 1); - if (arguments.length > 1) { - for (var i = 1; i < arguments.length; i++) { - args[i - 1] = arguments[i]; - } - } - queue.push(new Item(fun, args)); - if (queue.length === 1 && !draining) { - runTimeout(drainQueue); - } - } - // v8 likes predictible objects - function Item(fun, array) { - this.fun = fun; - this.array = array; - } - Item.prototype.run = function () { - this.fun.apply(null, this.array); - }; - - // from https://github.com/kumavis/browser-process-hrtime/blob/master/index.js - var performance = global$1.performance || {}; - performance.now || - performance.mozNow || - performance.msNow || - performance.oNow || - performance.webkitNow || - function(){ return (new Date()).getTime() }; - - var inherits; - if (typeof Object.create === 'function'){ - inherits = function inherits(ctor, superCtor) { - // implementation from standard node.js 'util' module - ctor.super_ = superCtor; - ctor.prototype = Object.create(superCtor.prototype, { - constructor: { - value: ctor, - enumerable: false, - writable: true, - configurable: true - } - }); - }; - } else { - inherits = function inherits(ctor, superCtor) { - ctor.super_ = superCtor; - var TempCtor = function () {}; - TempCtor.prototype = superCtor.prototype; - ctor.prototype = new TempCtor(); - ctor.prototype.constructor = ctor; - }; - } - var inherits$1 = inherits; - - var formatRegExp = /%[sdj%]/g; - function format(f) { - if (!isString(f)) { - var objects = []; - for (var i = 0; i < arguments.length; i++) { - objects.push(inspect(arguments[i])); - } - return objects.join(' '); - } - - var i = 1; - var args = arguments; - var len = args.length; - var str = String(f).replace(formatRegExp, function(x) { - if (x === '%%') return '%'; - if (i >= len) return x; - switch (x) { - case '%s': return String(args[i++]); - case '%d': return Number(args[i++]); - case '%j': - try { - return JSON.stringify(args[i++]); - } catch (_) { - return '[Circular]'; - } - default: - return x; - } - }); - for (var x = args[i]; i < len; x = args[++i]) { - if (isNull(x) || !isObject$1(x)) { - str += ' ' + x; - } else { - str += ' ' + inspect(x); - } - } - return str; - } - - // Mark that a method should not be used. - // Returns a modified function which warns once by default. - // If --no-deprecation is set, then it is a no-op. - function deprecate(fn, msg) { - // Allow for deprecating things in the process of starting up. - if (isUndefined(global$1.process)) { - return function() { - return deprecate(fn, msg).apply(this, arguments); - }; - } - - var warned = false; - function deprecated() { - if (!warned) { - { - console.error(msg); - } - warned = true; - } - return fn.apply(this, arguments); - } - - return deprecated; - } - - var debugs = {}; - var debugEnviron; - function debuglog(set) { - if (isUndefined(debugEnviron)) - debugEnviron = ''; - set = set.toUpperCase(); - if (!debugs[set]) { - if (new RegExp('\\b' + set + '\\b', 'i').test(debugEnviron)) { - var pid = 0; - debugs[set] = function() { - var msg = format.apply(null, arguments); - console.error('%s %d: %s', set, pid, msg); - }; - } else { - debugs[set] = function() {}; - } - } - return debugs[set]; - } - - /** - * Echos the value of a value. Trys to print the value out - * in the best way possible given the different types. - * - * @param {Object} obj The object to print out. - * @param {Object} opts Optional options object that alters the output. - */ - /* legacy: obj, showHidden, depth, colors*/ - function inspect(obj, opts) { - // default options - var ctx = { - seen: [], - stylize: stylizeNoColor - }; - // legacy... - if (arguments.length >= 3) ctx.depth = arguments[2]; - if (arguments.length >= 4) ctx.colors = arguments[3]; - if (isBoolean(opts)) { - // legacy... - ctx.showHidden = opts; - } else if (opts) { - // got an "options" object - _extend(ctx, opts); - } - // set default options - if (isUndefined(ctx.showHidden)) ctx.showHidden = false; - if (isUndefined(ctx.depth)) ctx.depth = 2; - if (isUndefined(ctx.colors)) ctx.colors = false; - if (isUndefined(ctx.customInspect)) ctx.customInspect = true; - if (ctx.colors) ctx.stylize = stylizeWithColor; - return formatValue(ctx, obj, ctx.depth); - } - - // http://en.wikipedia.org/wiki/ANSI_escape_code#graphics - inspect.colors = { - 'bold' : [1, 22], - 'italic' : [3, 23], - 'underline' : [4, 24], - 'inverse' : [7, 27], - 'white' : [37, 39], - 'grey' : [90, 39], - 'black' : [30, 39], - 'blue' : [34, 39], - 'cyan' : [36, 39], - 'green' : [32, 39], - 'magenta' : [35, 39], - 'red' : [31, 39], - 'yellow' : [33, 39] - }; - - // Don't use 'blue' not visible on cmd.exe - inspect.styles = { - 'special': 'cyan', - 'number': 'yellow', - 'boolean': 'yellow', - 'undefined': 'grey', - 'null': 'bold', - 'string': 'green', - 'date': 'magenta', - // "name": intentionally not styling - 'regexp': 'red' - }; - - - function stylizeWithColor(str, styleType) { - var style = inspect.styles[styleType]; - - if (style) { - return '\u001b[' + inspect.colors[style][0] + 'm' + str + - '\u001b[' + inspect.colors[style][1] + 'm'; - } else { - return str; - } - } - - - function stylizeNoColor(str, styleType) { - return str; - } - - - function arrayToHash(array) { - var hash = {}; - - array.forEach(function(val, idx) { - hash[val] = true; - }); - - return hash; - } - - - function formatValue(ctx, value, recurseTimes) { - // Provide a hook for user-specified inspect functions. - // Check that value is an object with an inspect function on it - if (ctx.customInspect && - value && - isFunction(value.inspect) && - // Filter out the util module, it's inspect function is special - value.inspect !== inspect && - // Also filter out any prototype objects using the circular check. - !(value.constructor && value.constructor.prototype === value)) { - var ret = value.inspect(recurseTimes, ctx); - if (!isString(ret)) { - ret = formatValue(ctx, ret, recurseTimes); - } - return ret; - } - - // Primitive types cannot have properties - var primitive = formatPrimitive(ctx, value); - if (primitive) { - return primitive; - } - - // Look up the keys of the object. - var keys = Object.keys(value); - var visibleKeys = arrayToHash(keys); - - if (ctx.showHidden) { - keys = Object.getOwnPropertyNames(value); - } - - // IE doesn't make error fields non-enumerable - // http://msdn.microsoft.com/en-us/library/ie/dww52sbt(v=vs.94).aspx - if (isError(value) - && (keys.indexOf('message') >= 0 || keys.indexOf('description') >= 0)) { - return formatError(value); - } - - // Some type of object without properties can be shortcutted. - if (keys.length === 0) { - if (isFunction(value)) { - var name = value.name ? ': ' + value.name : ''; - return ctx.stylize('[Function' + name + ']', 'special'); - } - if (isRegExp(value)) { - return ctx.stylize(RegExp.prototype.toString.call(value), 'regexp'); - } - if (isDate(value)) { - return ctx.stylize(Date.prototype.toString.call(value), 'date'); - } - if (isError(value)) { - return formatError(value); - } - } - - var base = '', array = false, braces = ['{', '}']; - - // Make Array say that they are Array - if (isArray(value)) { - array = true; - braces = ['[', ']']; - } - - // Make functions say that they are functions - if (isFunction(value)) { - var n = value.name ? ': ' + value.name : ''; - base = ' [Function' + n + ']'; - } - - // Make RegExps say that they are RegExps - if (isRegExp(value)) { - base = ' ' + RegExp.prototype.toString.call(value); - } - - // Make dates with properties first say the date - if (isDate(value)) { - base = ' ' + Date.prototype.toUTCString.call(value); - } - - // Make error with message first say the error - if (isError(value)) { - base = ' ' + formatError(value); - } - - if (keys.length === 0 && (!array || value.length == 0)) { - return braces[0] + base + braces[1]; - } - - if (recurseTimes < 0) { - if (isRegExp(value)) { - return ctx.stylize(RegExp.prototype.toString.call(value), 'regexp'); - } else { - return ctx.stylize('[Object]', 'special'); - } - } - - ctx.seen.push(value); - - var output; - if (array) { - output = formatArray(ctx, value, recurseTimes, visibleKeys, keys); - } else { - output = keys.map(function(key) { - return formatProperty(ctx, value, recurseTimes, visibleKeys, key, array); - }); - } - - ctx.seen.pop(); - - return reduceToSingleString(output, base, braces); - } - - - function formatPrimitive(ctx, value) { - if (isUndefined(value)) - return ctx.stylize('undefined', 'undefined'); - if (isString(value)) { - var simple = '\'' + JSON.stringify(value).replace(/^"|"$/g, '') - .replace(/'/g, "\\'") - .replace(/\\"/g, '"') + '\''; - return ctx.stylize(simple, 'string'); - } - if (isNumber(value)) - return ctx.stylize('' + value, 'number'); - if (isBoolean(value)) - return ctx.stylize('' + value, 'boolean'); - // For some reason typeof null is "object", so special case here. - if (isNull(value)) - return ctx.stylize('null', 'null'); - } - - - function formatError(value) { - return '[' + Error.prototype.toString.call(value) + ']'; - } - - - function formatArray(ctx, value, recurseTimes, visibleKeys, keys) { - var output = []; - for (var i = 0, l = value.length; i < l; ++i) { - if (hasOwnProperty(value, String(i))) { - output.push(formatProperty(ctx, value, recurseTimes, visibleKeys, - String(i), true)); - } else { - output.push(''); - } - } - keys.forEach(function(key) { - if (!key.match(/^\d+$/)) { - output.push(formatProperty(ctx, value, recurseTimes, visibleKeys, - key, true)); - } - }); - return output; - } - - - function formatProperty(ctx, value, recurseTimes, visibleKeys, key, array) { - var name, str, desc; - desc = Object.getOwnPropertyDescriptor(value, key) || { value: value[key] }; - if (desc.get) { - if (desc.set) { - str = ctx.stylize('[Getter/Setter]', 'special'); - } else { - str = ctx.stylize('[Getter]', 'special'); - } - } else { - if (desc.set) { - str = ctx.stylize('[Setter]', 'special'); - } - } - if (!hasOwnProperty(visibleKeys, key)) { - name = '[' + key + ']'; - } - if (!str) { - if (ctx.seen.indexOf(desc.value) < 0) { - if (isNull(recurseTimes)) { - str = formatValue(ctx, desc.value, null); - } else { - str = formatValue(ctx, desc.value, recurseTimes - 1); - } - if (str.indexOf('\n') > -1) { - if (array) { - str = str.split('\n').map(function(line) { - return ' ' + line; - }).join('\n').substr(2); - } else { - str = '\n' + str.split('\n').map(function(line) { - return ' ' + line; - }).join('\n'); - } - } - } else { - str = ctx.stylize('[Circular]', 'special'); - } - } - if (isUndefined(name)) { - if (array && key.match(/^\d+$/)) { - return str; - } - name = JSON.stringify('' + key); - if (name.match(/^"([a-zA-Z_][a-zA-Z_0-9]*)"$/)) { - name = name.substr(1, name.length - 2); - name = ctx.stylize(name, 'name'); - } else { - name = name.replace(/'/g, "\\'") - .replace(/\\"/g, '"') - .replace(/(^"|"$)/g, "'"); - name = ctx.stylize(name, 'string'); - } - } - - return name + ': ' + str; - } - - - function reduceToSingleString(output, base, braces) { - var length = output.reduce(function(prev, cur) { - if (cur.indexOf('\n') >= 0) ; - return prev + cur.replace(/\u001b\[\d\d?m/g, '').length + 1; - }, 0); - - if (length > 60) { - return braces[0] + - (base === '' ? '' : base + '\n ') + - ' ' + - output.join(',\n ') + - ' ' + - braces[1]; - } - - return braces[0] + base + ' ' + output.join(', ') + ' ' + braces[1]; - } - - - // NOTE: These type checking functions intentionally don't use `instanceof` - // because it is fragile and can be easily faked with `Object.create()`. - function isArray(ar) { - return Array.isArray(ar); - } - - function isBoolean(arg) { - return typeof arg === 'boolean'; - } - - function isNull(arg) { - return arg === null; - } - - function isNumber(arg) { - return typeof arg === 'number'; - } - - function isString(arg) { - return typeof arg === 'string'; - } - - function isUndefined(arg) { - return arg === void 0; - } - - function isRegExp(re) { - return isObject$1(re) && objectToString(re) === '[object RegExp]'; - } - - function isObject$1(arg) { - return typeof arg === 'object' && arg !== null; - } - - function isDate(d) { - return isObject$1(d) && objectToString(d) === '[object Date]'; - } - - function isError(e) { - return isObject$1(e) && - (objectToString(e) === '[object Error]' || e instanceof Error); - } - - function isFunction(arg) { - return typeof arg === 'function'; - } - - function objectToString(o) { - return Object.prototype.toString.call(o); - } - - function _extend(origin, add) { - // Don't do anything if add isn't an object - if (!add || !isObject$1(add)) return origin; - - var keys = Object.keys(add); - var i = keys.length; - while (i--) { - origin[keys[i]] = add[keys[i]]; - } - return origin; - } - function hasOwnProperty(obj, prop) { - return Object.prototype.hasOwnProperty.call(obj, prop); - } - - function BufferList() { - this.head = null; - this.tail = null; - this.length = 0; - } - - BufferList.prototype.push = function (v) { - var entry = { data: v, next: null }; - if (this.length > 0) this.tail.next = entry;else this.head = entry; - this.tail = entry; - ++this.length; - }; - - BufferList.prototype.unshift = function (v) { - var entry = { data: v, next: this.head }; - if (this.length === 0) this.tail = entry; - this.head = entry; - ++this.length; - }; - - BufferList.prototype.shift = function () { - if (this.length === 0) return; - var ret = this.head.data; - if (this.length === 1) this.head = this.tail = null;else this.head = this.head.next; - --this.length; - return ret; - }; - - BufferList.prototype.clear = function () { - this.head = this.tail = null; - this.length = 0; - }; - - BufferList.prototype.join = function (s) { - if (this.length === 0) return ''; - var p = this.head; - var ret = '' + p.data; - while (p = p.next) { - ret += s + p.data; - }return ret; - }; - - BufferList.prototype.concat = function (n) { - if (this.length === 0) return Buffer.alloc(0); - if (this.length === 1) return this.head.data; - var ret = Buffer.allocUnsafe(n >>> 0); - var p = this.head; - var i = 0; - while (p) { - p.data.copy(ret, i); - i += p.data.length; - p = p.next; - } - return ret; - }; - - // Copyright Joyent, Inc. and other Node contributors. - var isBufferEncoding = Buffer.isEncoding - || function(encoding) { - switch (encoding && encoding.toLowerCase()) { - case 'hex': case 'utf8': case 'utf-8': case 'ascii': case 'binary': case 'base64': case 'ucs2': case 'ucs-2': case 'utf16le': case 'utf-16le': case 'raw': return true; - default: return false; - } - }; - - - function assertEncoding(encoding) { - if (encoding && !isBufferEncoding(encoding)) { - throw new Error('Unknown encoding: ' + encoding); - } - } - - // StringDecoder provides an interface for efficiently splitting a series of - // buffers into a series of JS strings without breaking apart multi-byte - // characters. CESU-8 is handled as part of the UTF-8 encoding. - // - // @TODO Handling all encodings inside a single object makes it very difficult - // to reason about this code, so it should be split up in the future. - // @TODO There should be a utf8-strict encoding that rejects invalid UTF-8 code - // points as used by CESU-8. - function StringDecoder(encoding) { - this.encoding = (encoding || 'utf8').toLowerCase().replace(/[-_]/, ''); - assertEncoding(encoding); - switch (this.encoding) { - case 'utf8': - // CESU-8 represents each of Surrogate Pair by 3-bytes - this.surrogateSize = 3; - break; - case 'ucs2': - case 'utf16le': - // UTF-16 represents each of Surrogate Pair by 2-bytes - this.surrogateSize = 2; - this.detectIncompleteChar = utf16DetectIncompleteChar; - break; - case 'base64': - // Base-64 stores 3 bytes in 4 chars, and pads the remainder. - this.surrogateSize = 3; - this.detectIncompleteChar = base64DetectIncompleteChar; - break; - default: - this.write = passThroughWrite; - return; - } - - // Enough space to store all bytes of a single character. UTF-8 needs 4 - // bytes, but CESU-8 may require up to 6 (3 bytes per surrogate). - this.charBuffer = new Buffer(6); - // Number of bytes received for the current incomplete multi-byte character. - this.charReceived = 0; - // Number of bytes expected for the current incomplete multi-byte character. - this.charLength = 0; - } - - // write decodes the given buffer and returns it as JS string that is - // guaranteed to not contain any partial multi-byte characters. Any partial - // character found at the end of the buffer is buffered up, and will be - // returned when calling write again with the remaining bytes. - // - // Note: Converting a Buffer containing an orphan surrogate to a String - // currently works, but converting a String to a Buffer (via `new Buffer`, or - // Buffer#write) will replace incomplete surrogates with the unicode - // replacement character. See https://codereview.chromium.org/121173009/ . - StringDecoder.prototype.write = function(buffer) { - var charStr = ''; - // if our last write ended with an incomplete multibyte character - while (this.charLength) { - // determine how many remaining bytes this buffer has to offer for this char - var available = (buffer.length >= this.charLength - this.charReceived) ? - this.charLength - this.charReceived : - buffer.length; - - // add the new bytes to the char buffer - buffer.copy(this.charBuffer, this.charReceived, 0, available); - this.charReceived += available; - - if (this.charReceived < this.charLength) { - // still not enough chars in this buffer? wait for more ... - return ''; - } - - // remove bytes belonging to the current character from the buffer - buffer = buffer.slice(available, buffer.length); - - // get the character that was split - charStr = this.charBuffer.slice(0, this.charLength).toString(this.encoding); - - // CESU-8: lead surrogate (D800-DBFF) is also the incomplete character - var charCode = charStr.charCodeAt(charStr.length - 1); - if (charCode >= 0xD800 && charCode <= 0xDBFF) { - this.charLength += this.surrogateSize; - charStr = ''; - continue; - } - this.charReceived = this.charLength = 0; - - // if there are no more bytes in this buffer, just emit our char - if (buffer.length === 0) { - return charStr; - } - break; - } - - // determine and set charLength / charReceived - this.detectIncompleteChar(buffer); - - var end = buffer.length; - if (this.charLength) { - // buffer the incomplete character bytes we got - buffer.copy(this.charBuffer, 0, buffer.length - this.charReceived, end); - end -= this.charReceived; - } - - charStr += buffer.toString(this.encoding, 0, end); - - var end = charStr.length - 1; - var charCode = charStr.charCodeAt(end); - // CESU-8: lead surrogate (D800-DBFF) is also the incomplete character - if (charCode >= 0xD800 && charCode <= 0xDBFF) { - var size = this.surrogateSize; - this.charLength += size; - this.charReceived += size; - this.charBuffer.copy(this.charBuffer, size, 0, size); - buffer.copy(this.charBuffer, 0, 0, size); - return charStr.substring(0, end); - } - - // or just emit the charStr - return charStr; - }; - - // detectIncompleteChar determines if there is an incomplete UTF-8 character at - // the end of the given buffer. If so, it sets this.charLength to the byte - // length that character, and sets this.charReceived to the number of bytes - // that are available for this character. - StringDecoder.prototype.detectIncompleteChar = function(buffer) { - // determine how many bytes we have to check at the end of this buffer - var i = (buffer.length >= 3) ? 3 : buffer.length; - - // Figure out if one of the last i bytes of our buffer announces an - // incomplete char. - for (; i > 0; i--) { - var c = buffer[buffer.length - i]; - - // See http://en.wikipedia.org/wiki/UTF-8#Description - - // 110XXXXX - if (i == 1 && c >> 5 == 0x06) { - this.charLength = 2; - break; - } - - // 1110XXXX - if (i <= 2 && c >> 4 == 0x0E) { - this.charLength = 3; - break; - } - - // 11110XXX - if (i <= 3 && c >> 3 == 0x1E) { - this.charLength = 4; - break; - } - } - this.charReceived = i; - }; - - StringDecoder.prototype.end = function(buffer) { - var res = ''; - if (buffer && buffer.length) - res = this.write(buffer); - - if (this.charReceived) { - var cr = this.charReceived; - var buf = this.charBuffer; - var enc = this.encoding; - res += buf.slice(0, cr).toString(enc); - } - - return res; - }; - - function passThroughWrite(buffer) { - return buffer.toString(this.encoding); - } - - function utf16DetectIncompleteChar(buffer) { - this.charReceived = buffer.length % 2; - this.charLength = this.charReceived ? 2 : 0; - } - - function base64DetectIncompleteChar(buffer) { - this.charReceived = buffer.length % 3; - this.charLength = this.charReceived ? 3 : 0; - } - - Readable.ReadableState = ReadableState; - - var debug = debuglog('stream'); - inherits$1(Readable, EventEmitter); - - function prependListener(emitter, event, fn) { - // Sadly this is not cacheable as some libraries bundle their own - // event emitter implementation with them. - if (typeof emitter.prependListener === 'function') { - return emitter.prependListener(event, fn); - } else { - // This is a hack to make sure that our error handler is attached before any - // userland ones. NEVER DO THIS. This is here only because this code needs - // to continue to work with older versions of Node.js that do not include - // the prependListener() method. The goal is to eventually remove this hack. - if (!emitter._events || !emitter._events[event]) - emitter.on(event, fn); - else if (Array.isArray(emitter._events[event])) - emitter._events[event].unshift(fn); - else - emitter._events[event] = [fn, emitter._events[event]]; - } - } - function listenerCount (emitter, type) { - return emitter.listeners(type).length; - } - function ReadableState(options, stream) { - - options = options || {}; - - // object stream flag. Used to make read(n) ignore n and to - // make all the buffer merging and length checks go away - this.objectMode = !!options.objectMode; - - if (stream instanceof Duplex) this.objectMode = this.objectMode || !!options.readableObjectMode; - - // the point at which it stops calling _read() to fill the buffer - // Note: 0 is a valid value, means "don't call _read preemptively ever" - var hwm = options.highWaterMark; - var defaultHwm = this.objectMode ? 16 : 16 * 1024; - this.highWaterMark = hwm || hwm === 0 ? hwm : defaultHwm; - - // cast to ints. - this.highWaterMark = ~ ~this.highWaterMark; - - // A linked list is used to store data chunks instead of an array because the - // linked list can remove elements from the beginning faster than - // array.shift() - this.buffer = new BufferList(); - this.length = 0; - this.pipes = null; - this.pipesCount = 0; - this.flowing = null; - this.ended = false; - this.endEmitted = false; - this.reading = false; - - // a flag to be able to tell if the onwrite cb is called immediately, - // or on a later tick. We set this to true at first, because any - // actions that shouldn't happen until "later" should generally also - // not happen before the first write call. - this.sync = true; - - // whenever we return null, then we set a flag to say - // that we're awaiting a 'readable' event emission. - this.needReadable = false; - this.emittedReadable = false; - this.readableListening = false; - this.resumeScheduled = false; - - // Crypto is kind of old and crusty. Historically, its default string - // encoding is 'binary' so we have to make this configurable. - // Everything else in the universe uses 'utf8', though. - this.defaultEncoding = options.defaultEncoding || 'utf8'; - - // when piping, we only care about 'readable' events that happen - // after read()ing all the bytes and not getting any pushback. - this.ranOut = false; - - // the number of writers that are awaiting a drain event in .pipe()s - this.awaitDrain = 0; - - // if true, a maybeReadMore has been scheduled - this.readingMore = false; - - this.decoder = null; - this.encoding = null; - if (options.encoding) { - this.decoder = new StringDecoder(options.encoding); - this.encoding = options.encoding; - } - } - function Readable(options) { - - if (!(this instanceof Readable)) return new Readable(options); - - this._readableState = new ReadableState(options, this); - - // legacy - this.readable = true; - - if (options && typeof options.read === 'function') this._read = options.read; - - EventEmitter.call(this); - } - - // Manually shove something into the read() buffer. - // This returns true if the highWaterMark has not been hit yet, - // similar to how Writable.write() returns true if you should - // write() some more. - Readable.prototype.push = function (chunk, encoding) { - var state = this._readableState; - - if (!state.objectMode && typeof chunk === 'string') { - encoding = encoding || state.defaultEncoding; - if (encoding !== state.encoding) { - chunk = Buffer.from(chunk, encoding); - encoding = ''; - } - } - - return readableAddChunk(this, state, chunk, encoding, false); - }; - - // Unshift should *always* be something directly out of read() - Readable.prototype.unshift = function (chunk) { - var state = this._readableState; - return readableAddChunk(this, state, chunk, '', true); - }; - - Readable.prototype.isPaused = function () { - return this._readableState.flowing === false; - }; - - function readableAddChunk(stream, state, chunk, encoding, addToFront) { - var er = chunkInvalid(state, chunk); - if (er) { - stream.emit('error', er); - } else if (chunk === null) { - state.reading = false; - onEofChunk(stream, state); - } else if (state.objectMode || chunk && chunk.length > 0) { - if (state.ended && !addToFront) { - var e = new Error('stream.push() after EOF'); - stream.emit('error', e); - } else if (state.endEmitted && addToFront) { - var _e = new Error('stream.unshift() after end event'); - stream.emit('error', _e); - } else { - var skipAdd; - if (state.decoder && !addToFront && !encoding) { - chunk = state.decoder.write(chunk); - skipAdd = !state.objectMode && chunk.length === 0; - } - - if (!addToFront) state.reading = false; - - // Don't add to the buffer if we've decoded to an empty string chunk and - // we're not in object mode - if (!skipAdd) { - // if we want the data now, just emit it. - if (state.flowing && state.length === 0 && !state.sync) { - stream.emit('data', chunk); - stream.read(0); - } else { - // update the buffer info. - state.length += state.objectMode ? 1 : chunk.length; - if (addToFront) state.buffer.unshift(chunk);else state.buffer.push(chunk); - - if (state.needReadable) emitReadable(stream); - } - } - - maybeReadMore(stream, state); - } - } else if (!addToFront) { - state.reading = false; - } - - return needMoreData(state); - } - - // if it's past the high water mark, we can push in some more. - // Also, if we have no data yet, we can stand some - // more bytes. This is to work around cases where hwm=0, - // such as the repl. Also, if the push() triggered a - // readable event, and the user called read(largeNumber) such that - // needReadable was set, then we ought to push more, so that another - // 'readable' event will be triggered. - function needMoreData(state) { - return !state.ended && (state.needReadable || state.length < state.highWaterMark || state.length === 0); - } - - // backwards compatibility. - Readable.prototype.setEncoding = function (enc) { - this._readableState.decoder = new StringDecoder(enc); - this._readableState.encoding = enc; - return this; - }; - - // Don't raise the hwm > 8MB - var MAX_HWM = 0x800000; - function computeNewHighWaterMark(n) { - if (n >= MAX_HWM) { - n = MAX_HWM; - } else { - // Get the next highest power of 2 to prevent increasing hwm excessively in - // tiny amounts - n--; - n |= n >>> 1; - n |= n >>> 2; - n |= n >>> 4; - n |= n >>> 8; - n |= n >>> 16; - n++; - } - return n; - } - - // This function is designed to be inlinable, so please take care when making - // changes to the function body. - function howMuchToRead(n, state) { - if (n <= 0 || state.length === 0 && state.ended) return 0; - if (state.objectMode) return 1; - if (n !== n) { - // Only flow one buffer at a time - if (state.flowing && state.length) return state.buffer.head.data.length;else return state.length; - } - // If we're asking for more than the current hwm, then raise the hwm. - if (n > state.highWaterMark) state.highWaterMark = computeNewHighWaterMark(n); - if (n <= state.length) return n; - // Don't have enough - if (!state.ended) { - state.needReadable = true; - return 0; - } - return state.length; - } - - // you can override either this method, or the async _read(n) below. - Readable.prototype.read = function (n) { - debug('read', n); - n = parseInt(n, 10); - var state = this._readableState; - var nOrig = n; - - if (n !== 0) state.emittedReadable = false; - - // if we're doing read(0) to trigger a readable event, but we - // already have a bunch of data in the buffer, then just trigger - // the 'readable' event and move on. - if (n === 0 && state.needReadable && (state.length >= state.highWaterMark || state.ended)) { - debug('read: emitReadable', state.length, state.ended); - if (state.length === 0 && state.ended) endReadable(this);else emitReadable(this); - return null; - } - - n = howMuchToRead(n, state); - - // if we've ended, and we're now clear, then finish it up. - if (n === 0 && state.ended) { - if (state.length === 0) endReadable(this); - return null; - } - - // All the actual chunk generation logic needs to be - // *below* the call to _read. The reason is that in certain - // synthetic stream cases, such as passthrough streams, _read - // may be a completely synchronous operation which may change - // the state of the read buffer, providing enough data when - // before there was *not* enough. - // - // So, the steps are: - // 1. Figure out what the state of things will be after we do - // a read from the buffer. - // - // 2. If that resulting state will trigger a _read, then call _read. - // Note that this may be asynchronous, or synchronous. Yes, it is - // deeply ugly to write APIs this way, but that still doesn't mean - // that the Readable class should behave improperly, as streams are - // designed to be sync/async agnostic. - // Take note if the _read call is sync or async (ie, if the read call - // has returned yet), so that we know whether or not it's safe to emit - // 'readable' etc. - // - // 3. Actually pull the requested chunks out of the buffer and return. - - // if we need a readable event, then we need to do some reading. - var doRead = state.needReadable; - debug('need readable', doRead); - - // if we currently have less than the highWaterMark, then also read some - if (state.length === 0 || state.length - n < state.highWaterMark) { - doRead = true; - debug('length less than watermark', doRead); - } - - // however, if we've ended, then there's no point, and if we're already - // reading, then it's unnecessary. - if (state.ended || state.reading) { - doRead = false; - debug('reading or ended', doRead); - } else if (doRead) { - debug('do read'); - state.reading = true; - state.sync = true; - // if the length is currently zero, then we *need* a readable event. - if (state.length === 0) state.needReadable = true; - // call internal read method - this._read(state.highWaterMark); - state.sync = false; - // If _read pushed data synchronously, then `reading` will be false, - // and we need to re-evaluate how much data we can return to the user. - if (!state.reading) n = howMuchToRead(nOrig, state); - } - - var ret; - if (n > 0) ret = fromList(n, state);else ret = null; - - if (ret === null) { - state.needReadable = true; - n = 0; - } else { - state.length -= n; - } - - if (state.length === 0) { - // If we have nothing in the buffer, then we want to know - // as soon as we *do* get something into the buffer. - if (!state.ended) state.needReadable = true; - - // If we tried to read() past the EOF, then emit end on the next tick. - if (nOrig !== n && state.ended) endReadable(this); - } - - if (ret !== null) this.emit('data', ret); - - return ret; - }; - - function chunkInvalid(state, chunk) { - var er = null; - if (!isBuffer(chunk) && typeof chunk !== 'string' && chunk !== null && chunk !== undefined && !state.objectMode) { - er = new TypeError('Invalid non-string/buffer chunk'); - } - return er; - } - - function onEofChunk(stream, state) { - if (state.ended) return; - if (state.decoder) { - var chunk = state.decoder.end(); - if (chunk && chunk.length) { - state.buffer.push(chunk); - state.length += state.objectMode ? 1 : chunk.length; - } - } - state.ended = true; - - // emit 'readable' now to make sure it gets picked up. - emitReadable(stream); - } - - // Don't emit readable right away in sync mode, because this can trigger - // another read() call => stack overflow. This way, it might trigger - // a nextTick recursion warning, but that's not so bad. - function emitReadable(stream) { - var state = stream._readableState; - state.needReadable = false; - if (!state.emittedReadable) { - debug('emitReadable', state.flowing); - state.emittedReadable = true; - if (state.sync) nextTick(emitReadable_, stream);else emitReadable_(stream); - } - } - - function emitReadable_(stream) { - debug('emit readable'); - stream.emit('readable'); - flow(stream); - } - - // at this point, the user has presumably seen the 'readable' event, - // and called read() to consume some data. that may have triggered - // in turn another _read(n) call, in which case reading = true if - // it's in progress. - // However, if we're not ended, or reading, and the length < hwm, - // then go ahead and try to read some more preemptively. - function maybeReadMore(stream, state) { - if (!state.readingMore) { - state.readingMore = true; - nextTick(maybeReadMore_, stream, state); - } - } - - function maybeReadMore_(stream, state) { - var len = state.length; - while (!state.reading && !state.flowing && !state.ended && state.length < state.highWaterMark) { - debug('maybeReadMore read 0'); - stream.read(0); - if (len === state.length) - // didn't get any data, stop spinning. - break;else len = state.length; - } - state.readingMore = false; - } - - // abstract method. to be overridden in specific implementation classes. - // call cb(er, data) where data is <= n in length. - // for virtual (non-string, non-buffer) streams, "length" is somewhat - // arbitrary, and perhaps not very meaningful. - Readable.prototype._read = function (n) { - this.emit('error', new Error('not implemented')); - }; - - Readable.prototype.pipe = function (dest, pipeOpts) { - var src = this; - var state = this._readableState; - - switch (state.pipesCount) { - case 0: - state.pipes = dest; - break; - case 1: - state.pipes = [state.pipes, dest]; - break; - default: - state.pipes.push(dest); - break; - } - state.pipesCount += 1; - debug('pipe count=%d opts=%j', state.pipesCount, pipeOpts); - - var doEnd = (!pipeOpts || pipeOpts.end !== false); - - var endFn = doEnd ? onend : cleanup; - if (state.endEmitted) nextTick(endFn);else src.once('end', endFn); - - dest.on('unpipe', onunpipe); - function onunpipe(readable) { - debug('onunpipe'); - if (readable === src) { - cleanup(); - } - } - - function onend() { - debug('onend'); - dest.end(); - } - - // when the dest drains, it reduces the awaitDrain counter - // on the source. This would be more elegant with a .once() - // handler in flow(), but adding and removing repeatedly is - // too slow. - var ondrain = pipeOnDrain(src); - dest.on('drain', ondrain); - - var cleanedUp = false; - function cleanup() { - debug('cleanup'); - // cleanup event handlers once the pipe is broken - dest.removeListener('close', onclose); - dest.removeListener('finish', onfinish); - dest.removeListener('drain', ondrain); - dest.removeListener('error', onerror); - dest.removeListener('unpipe', onunpipe); - src.removeListener('end', onend); - src.removeListener('end', cleanup); - src.removeListener('data', ondata); - - cleanedUp = true; - - // if the reader is waiting for a drain event from this - // specific writer, then it would cause it to never start - // flowing again. - // So, if this is awaiting a drain, then we just call it now. - // If we don't know, then assume that we are waiting for one. - if (state.awaitDrain && (!dest._writableState || dest._writableState.needDrain)) ondrain(); - } - - // If the user pushes more data while we're writing to dest then we'll end up - // in ondata again. However, we only want to increase awaitDrain once because - // dest will only emit one 'drain' event for the multiple writes. - // => Introduce a guard on increasing awaitDrain. - var increasedAwaitDrain = false; - src.on('data', ondata); - function ondata(chunk) { - debug('ondata'); - increasedAwaitDrain = false; - var ret = dest.write(chunk); - if (false === ret && !increasedAwaitDrain) { - // If the user unpiped during `dest.write()`, it is possible - // to get stuck in a permanently paused state if that write - // also returned false. - // => Check whether `dest` is still a piping destination. - if ((state.pipesCount === 1 && state.pipes === dest || state.pipesCount > 1 && indexOf(state.pipes, dest) !== -1) && !cleanedUp) { - debug('false write response, pause', src._readableState.awaitDrain); - src._readableState.awaitDrain++; - increasedAwaitDrain = true; - } - src.pause(); - } - } - - // if the dest has an error, then stop piping into it. - // however, don't suppress the throwing behavior for this. - function onerror(er) { - debug('onerror', er); - unpipe(); - dest.removeListener('error', onerror); - if (listenerCount(dest, 'error') === 0) dest.emit('error', er); - } - - // Make sure our error handler is attached before userland ones. - prependListener(dest, 'error', onerror); - - // Both close and finish should trigger unpipe, but only once. - function onclose() { - dest.removeListener('finish', onfinish); - unpipe(); - } - dest.once('close', onclose); - function onfinish() { - debug('onfinish'); - dest.removeListener('close', onclose); - unpipe(); - } - dest.once('finish', onfinish); - - function unpipe() { - debug('unpipe'); - src.unpipe(dest); - } - - // tell the dest that it's being piped to - dest.emit('pipe', src); - - // start the flow if it hasn't been started already. - if (!state.flowing) { - debug('pipe resume'); - src.resume(); - } - - return dest; - }; - - function pipeOnDrain(src) { - return function () { - var state = src._readableState; - debug('pipeOnDrain', state.awaitDrain); - if (state.awaitDrain) state.awaitDrain--; - if (state.awaitDrain === 0 && src.listeners('data').length) { - state.flowing = true; - flow(src); - } - }; - } - - Readable.prototype.unpipe = function (dest) { - var state = this._readableState; - - // if we're not piping anywhere, then do nothing. - if (state.pipesCount === 0) return this; - - // just one destination. most common case. - if (state.pipesCount === 1) { - // passed in one, but it's not the right one. - if (dest && dest !== state.pipes) return this; - - if (!dest) dest = state.pipes; - - // got a match. - state.pipes = null; - state.pipesCount = 0; - state.flowing = false; - if (dest) dest.emit('unpipe', this); - return this; - } - - // slow case. multiple pipe destinations. - - if (!dest) { - // remove all. - var dests = state.pipes; - var len = state.pipesCount; - state.pipes = null; - state.pipesCount = 0; - state.flowing = false; - - for (var _i = 0; _i < len; _i++) { - dests[_i].emit('unpipe', this); - }return this; - } - - // try to find the right one. - var i = indexOf(state.pipes, dest); - if (i === -1) return this; - - state.pipes.splice(i, 1); - state.pipesCount -= 1; - if (state.pipesCount === 1) state.pipes = state.pipes[0]; - - dest.emit('unpipe', this); - - return this; - }; - - // set up data events if they are asked for - // Ensure readable listeners eventually get something - Readable.prototype.on = function (ev, fn) { - var res = EventEmitter.prototype.on.call(this, ev, fn); - - if (ev === 'data') { - // Start flowing on next tick if stream isn't explicitly paused - if (this._readableState.flowing !== false) this.resume(); - } else if (ev === 'readable') { - var state = this._readableState; - if (!state.endEmitted && !state.readableListening) { - state.readableListening = state.needReadable = true; - state.emittedReadable = false; - if (!state.reading) { - nextTick(nReadingNextTick, this); - } else if (state.length) { - emitReadable(this); - } - } - } - - return res; - }; - Readable.prototype.addListener = Readable.prototype.on; - - function nReadingNextTick(self) { - debug('readable nexttick read 0'); - self.read(0); - } - - // pause() and resume() are remnants of the legacy readable stream API - // If the user uses them, then switch into old mode. - Readable.prototype.resume = function () { - var state = this._readableState; - if (!state.flowing) { - debug('resume'); - state.flowing = true; - resume(this, state); - } - return this; - }; - - function resume(stream, state) { - if (!state.resumeScheduled) { - state.resumeScheduled = true; - nextTick(resume_, stream, state); - } - } - - function resume_(stream, state) { - if (!state.reading) { - debug('resume read 0'); - stream.read(0); - } - - state.resumeScheduled = false; - state.awaitDrain = 0; - stream.emit('resume'); - flow(stream); - if (state.flowing && !state.reading) stream.read(0); - } - - Readable.prototype.pause = function () { - debug('call pause flowing=%j', this._readableState.flowing); - if (false !== this._readableState.flowing) { - debug('pause'); - this._readableState.flowing = false; - this.emit('pause'); - } - return this; - }; - - function flow(stream) { - var state = stream._readableState; - debug('flow', state.flowing); - while (state.flowing && stream.read() !== null) {} - } - - // wrap an old-style stream as the async data source. - // This is *not* part of the readable stream interface. - // It is an ugly unfortunate mess of history. - Readable.prototype.wrap = function (stream) { - var state = this._readableState; - var paused = false; - - var self = this; - stream.on('end', function () { - debug('wrapped end'); - if (state.decoder && !state.ended) { - var chunk = state.decoder.end(); - if (chunk && chunk.length) self.push(chunk); - } - - self.push(null); - }); - - stream.on('data', function (chunk) { - debug('wrapped data'); - if (state.decoder) chunk = state.decoder.write(chunk); - - // don't skip over falsy values in objectMode - if (state.objectMode && (chunk === null || chunk === undefined)) return;else if (!state.objectMode && (!chunk || !chunk.length)) return; - - var ret = self.push(chunk); - if (!ret) { - paused = true; - stream.pause(); - } - }); - - // proxy all the other methods. - // important when wrapping filters and duplexes. - for (var i in stream) { - if (this[i] === undefined && typeof stream[i] === 'function') { - this[i] = function (method) { - return function () { - return stream[method].apply(stream, arguments); - }; - }(i); - } - } - - // proxy certain important events. - var events = ['error', 'close', 'destroy', 'pause', 'resume']; - forEach(events, function (ev) { - stream.on(ev, self.emit.bind(self, ev)); - }); - - // when we try to consume some more bytes, simply unpause the - // underlying stream. - self._read = function (n) { - debug('wrapped _read', n); - if (paused) { - paused = false; - stream.resume(); - } - }; - - return self; - }; - - // exposed for testing purposes only. - Readable._fromList = fromList; - - // Pluck off n bytes from an array of buffers. - // Length is the combined lengths of all the buffers in the list. - // This function is designed to be inlinable, so please take care when making - // changes to the function body. - function fromList(n, state) { - // nothing buffered - if (state.length === 0) return null; - - var ret; - if (state.objectMode) ret = state.buffer.shift();else if (!n || n >= state.length) { - // read it all, truncate the list - if (state.decoder) ret = state.buffer.join('');else if (state.buffer.length === 1) ret = state.buffer.head.data;else ret = state.buffer.concat(state.length); - state.buffer.clear(); - } else { - // read part of list - ret = fromListPartial(n, state.buffer, state.decoder); - } - - return ret; - } - - // Extracts only enough buffered data to satisfy the amount requested. - // This function is designed to be inlinable, so please take care when making - // changes to the function body. - function fromListPartial(n, list, hasStrings) { - var ret; - if (n < list.head.data.length) { - // slice is the same for buffers and strings - ret = list.head.data.slice(0, n); - list.head.data = list.head.data.slice(n); - } else if (n === list.head.data.length) { - // first chunk is a perfect match - ret = list.shift(); - } else { - // result spans more than one buffer - ret = hasStrings ? copyFromBufferString(n, list) : copyFromBuffer(n, list); - } - return ret; - } - - // Copies a specified amount of characters from the list of buffered data - // chunks. - // This function is designed to be inlinable, so please take care when making - // changes to the function body. - function copyFromBufferString(n, list) { - var p = list.head; - var c = 1; - var ret = p.data; - n -= ret.length; - while (p = p.next) { - var str = p.data; - var nb = n > str.length ? str.length : n; - if (nb === str.length) ret += str;else ret += str.slice(0, n); - n -= nb; - if (n === 0) { - if (nb === str.length) { - ++c; - if (p.next) list.head = p.next;else list.head = list.tail = null; - } else { - list.head = p; - p.data = str.slice(nb); - } - break; - } - ++c; - } - list.length -= c; - return ret; - } - - // Copies a specified amount of bytes from the list of buffered data chunks. - // This function is designed to be inlinable, so please take care when making - // changes to the function body. - function copyFromBuffer(n, list) { - var ret = Buffer.allocUnsafe(n); - var p = list.head; - var c = 1; - p.data.copy(ret); - n -= p.data.length; - while (p = p.next) { - var buf = p.data; - var nb = n > buf.length ? buf.length : n; - buf.copy(ret, ret.length - n, 0, nb); - n -= nb; - if (n === 0) { - if (nb === buf.length) { - ++c; - if (p.next) list.head = p.next;else list.head = list.tail = null; - } else { - list.head = p; - p.data = buf.slice(nb); - } - break; - } - ++c; - } - list.length -= c; - return ret; - } - - function endReadable(stream) { - var state = stream._readableState; - - // If we get here before consuming all the bytes, then that is a - // bug in node. Should never happen. - if (state.length > 0) throw new Error('"endReadable()" called on non-empty stream'); - - if (!state.endEmitted) { - state.ended = true; - nextTick(endReadableNT, state, stream); - } - } - - function endReadableNT(state, stream) { - // Check that we didn't get one last unshift. - if (!state.endEmitted && state.length === 0) { - state.endEmitted = true; - stream.readable = false; - stream.emit('end'); - } - } - - function forEach(xs, f) { - for (var i = 0, l = xs.length; i < l; i++) { - f(xs[i], i); - } - } - - function indexOf(xs, x) { - for (var i = 0, l = xs.length; i < l; i++) { - if (xs[i] === x) return i; - } - return -1; - } - - // A bit simpler than readable streams. - Writable.WritableState = WritableState; - inherits$1(Writable, EventEmitter); - - function nop() {} - - function WriteReq(chunk, encoding, cb) { - this.chunk = chunk; - this.encoding = encoding; - this.callback = cb; - this.next = null; - } - - function WritableState(options, stream) { - Object.defineProperty(this, 'buffer', { - get: deprecate(function () { - return this.getBuffer(); - }, '_writableState.buffer is deprecated. Use _writableState.getBuffer ' + 'instead.') - }); - options = options || {}; - - // object stream flag to indicate whether or not this stream - // contains buffers or objects. - this.objectMode = !!options.objectMode; - - if (stream instanceof Duplex) this.objectMode = this.objectMode || !!options.writableObjectMode; - - // the point at which write() starts returning false - // Note: 0 is a valid value, means that we always return false if - // the entire buffer is not flushed immediately on write() - var hwm = options.highWaterMark; - var defaultHwm = this.objectMode ? 16 : 16 * 1024; - this.highWaterMark = hwm || hwm === 0 ? hwm : defaultHwm; - - // cast to ints. - this.highWaterMark = ~ ~this.highWaterMark; - - this.needDrain = false; - // at the start of calling end() - this.ending = false; - // when end() has been called, and returned - this.ended = false; - // when 'finish' is emitted - this.finished = false; - - // should we decode strings into buffers before passing to _write? - // this is here so that some node-core streams can optimize string - // handling at a lower level. - var noDecode = options.decodeStrings === false; - this.decodeStrings = !noDecode; - - // Crypto is kind of old and crusty. Historically, its default string - // encoding is 'binary' so we have to make this configurable. - // Everything else in the universe uses 'utf8', though. - this.defaultEncoding = options.defaultEncoding || 'utf8'; - - // not an actual buffer we keep track of, but a measurement - // of how much we're waiting to get pushed to some underlying - // socket or file. - this.length = 0; - - // a flag to see when we're in the middle of a write. - this.writing = false; - - // when true all writes will be buffered until .uncork() call - this.corked = 0; - - // a flag to be able to tell if the onwrite cb is called immediately, - // or on a later tick. We set this to true at first, because any - // actions that shouldn't happen until "later" should generally also - // not happen before the first write call. - this.sync = true; - - // a flag to know if we're processing previously buffered items, which - // may call the _write() callback in the same tick, so that we don't - // end up in an overlapped onwrite situation. - this.bufferProcessing = false; - - // the callback that's passed to _write(chunk,cb) - this.onwrite = function (er) { - onwrite(stream, er); - }; - - // the callback that the user supplies to write(chunk,encoding,cb) - this.writecb = null; - - // the amount that is being written when _write is called. - this.writelen = 0; - - this.bufferedRequest = null; - this.lastBufferedRequest = null; - - // number of pending user-supplied write callbacks - // this must be 0 before 'finish' can be emitted - this.pendingcb = 0; - - // emit prefinish if the only thing we're waiting for is _write cbs - // This is relevant for synchronous Transform streams - this.prefinished = false; - - // True if the error was already emitted and should not be thrown again - this.errorEmitted = false; - - // count buffered requests - this.bufferedRequestCount = 0; - - // allocate the first CorkedRequest, there is always - // one allocated and free to use, and we maintain at most two - this.corkedRequestsFree = new CorkedRequest(this); - } - - WritableState.prototype.getBuffer = function writableStateGetBuffer() { - var current = this.bufferedRequest; - var out = []; - while (current) { - out.push(current); - current = current.next; - } - return out; - }; - function Writable(options) { - - // Writable ctor is applied to Duplexes, though they're not - // instanceof Writable, they're instanceof Readable. - if (!(this instanceof Writable) && !(this instanceof Duplex)) return new Writable(options); - - this._writableState = new WritableState(options, this); - - // legacy. - this.writable = true; - - if (options) { - if (typeof options.write === 'function') this._write = options.write; - - if (typeof options.writev === 'function') this._writev = options.writev; - } - - EventEmitter.call(this); - } - - // Otherwise people can pipe Writable streams, which is just wrong. - Writable.prototype.pipe = function () { - this.emit('error', new Error('Cannot pipe, not readable')); - }; - - function writeAfterEnd(stream, cb) { - var er = new Error('write after end'); - // TODO: defer error events consistently everywhere, not just the cb - stream.emit('error', er); - nextTick(cb, er); - } - - // If we get something that is not a buffer, string, null, or undefined, - // and we're not in objectMode, then that's an error. - // Otherwise stream chunks are all considered to be of length=1, and the - // watermarks determine how many objects to keep in the buffer, rather than - // how many bytes or characters. - function validChunk(stream, state, chunk, cb) { - var valid = true; - var er = false; - // Always throw error if a null is written - // if we are not in object mode then throw - // if it is not a buffer, string, or undefined. - if (chunk === null) { - er = new TypeError('May not write null values to stream'); - } else if (!Buffer.isBuffer(chunk) && typeof chunk !== 'string' && chunk !== undefined && !state.objectMode) { - er = new TypeError('Invalid non-string/buffer chunk'); - } - if (er) { - stream.emit('error', er); - nextTick(cb, er); - valid = false; - } - return valid; - } - - Writable.prototype.write = function (chunk, encoding, cb) { - var state = this._writableState; - var ret = false; - - if (typeof encoding === 'function') { - cb = encoding; - encoding = null; - } - - if (Buffer.isBuffer(chunk)) encoding = 'buffer';else if (!encoding) encoding = state.defaultEncoding; - - if (typeof cb !== 'function') cb = nop; - - if (state.ended) writeAfterEnd(this, cb);else if (validChunk(this, state, chunk, cb)) { - state.pendingcb++; - ret = writeOrBuffer(this, state, chunk, encoding, cb); - } - - return ret; - }; - - Writable.prototype.cork = function () { - var state = this._writableState; - - state.corked++; - }; - - Writable.prototype.uncork = function () { - var state = this._writableState; - - if (state.corked) { - state.corked--; - - if (!state.writing && !state.corked && !state.finished && !state.bufferProcessing && state.bufferedRequest) clearBuffer(this, state); - } - }; - - Writable.prototype.setDefaultEncoding = function setDefaultEncoding(encoding) { - // node::ParseEncoding() requires lower case. - if (typeof encoding === 'string') encoding = encoding.toLowerCase(); - if (!(['hex', 'utf8', 'utf-8', 'ascii', 'binary', 'base64', 'ucs2', 'ucs-2', 'utf16le', 'utf-16le', 'raw'].indexOf((encoding + '').toLowerCase()) > -1)) throw new TypeError('Unknown encoding: ' + encoding); - this._writableState.defaultEncoding = encoding; - return this; - }; - - function decodeChunk(state, chunk, encoding) { - if (!state.objectMode && state.decodeStrings !== false && typeof chunk === 'string') { - chunk = Buffer.from(chunk, encoding); - } - return chunk; - } - - // if we're already writing something, then just put this - // in the queue, and wait our turn. Otherwise, call _write - // If we return false, then we need a drain event, so set that flag. - function writeOrBuffer(stream, state, chunk, encoding, cb) { - chunk = decodeChunk(state, chunk, encoding); - - if (Buffer.isBuffer(chunk)) encoding = 'buffer'; - var len = state.objectMode ? 1 : chunk.length; - - state.length += len; - - var ret = state.length < state.highWaterMark; - // we must ensure that previous needDrain will not be reset to false. - if (!ret) state.needDrain = true; - - if (state.writing || state.corked) { - var last = state.lastBufferedRequest; - state.lastBufferedRequest = new WriteReq(chunk, encoding, cb); - if (last) { - last.next = state.lastBufferedRequest; - } else { - state.bufferedRequest = state.lastBufferedRequest; - } - state.bufferedRequestCount += 1; - } else { - doWrite(stream, state, false, len, chunk, encoding, cb); - } - - return ret; - } - - function doWrite(stream, state, writev, len, chunk, encoding, cb) { - state.writelen = len; - state.writecb = cb; - state.writing = true; - state.sync = true; - if (writev) stream._writev(chunk, state.onwrite);else stream._write(chunk, encoding, state.onwrite); - state.sync = false; - } - - function onwriteError(stream, state, sync, er, cb) { - --state.pendingcb; - if (sync) nextTick(cb, er);else cb(er); - - stream._writableState.errorEmitted = true; - stream.emit('error', er); - } - - function onwriteStateUpdate(state) { - state.writing = false; - state.writecb = null; - state.length -= state.writelen; - state.writelen = 0; - } - - function onwrite(stream, er) { - var state = stream._writableState; - var sync = state.sync; - var cb = state.writecb; - - onwriteStateUpdate(state); - - if (er) onwriteError(stream, state, sync, er, cb);else { - // Check if we're actually ready to finish, but don't emit yet - var finished = needFinish(state); - - if (!finished && !state.corked && !state.bufferProcessing && state.bufferedRequest) { - clearBuffer(stream, state); - } - - if (sync) { - /**/ - nextTick(afterWrite, stream, state, finished, cb); - /**/ - } else { - afterWrite(stream, state, finished, cb); - } - } - } - - function afterWrite(stream, state, finished, cb) { - if (!finished) onwriteDrain(stream, state); - state.pendingcb--; - cb(); - finishMaybe(stream, state); - } - - // Must force callback to be called on nextTick, so that we don't - // emit 'drain' before the write() consumer gets the 'false' return - // value, and has a chance to attach a 'drain' listener. - function onwriteDrain(stream, state) { - if (state.length === 0 && state.needDrain) { - state.needDrain = false; - stream.emit('drain'); - } - } - - // if there's something in the buffer waiting, then process it - function clearBuffer(stream, state) { - state.bufferProcessing = true; - var entry = state.bufferedRequest; - - if (stream._writev && entry && entry.next) { - // Fast case, write everything using _writev() - var l = state.bufferedRequestCount; - var buffer = new Array(l); - var holder = state.corkedRequestsFree; - holder.entry = entry; - - var count = 0; - while (entry) { - buffer[count] = entry; - entry = entry.next; - count += 1; - } - - doWrite(stream, state, true, state.length, buffer, '', holder.finish); - - // doWrite is almost always async, defer these to save a bit of time - // as the hot path ends with doWrite - state.pendingcb++; - state.lastBufferedRequest = null; - if (holder.next) { - state.corkedRequestsFree = holder.next; - holder.next = null; - } else { - state.corkedRequestsFree = new CorkedRequest(state); - } - } else { - // Slow case, write chunks one-by-one - while (entry) { - var chunk = entry.chunk; - var encoding = entry.encoding; - var cb = entry.callback; - var len = state.objectMode ? 1 : chunk.length; - - doWrite(stream, state, false, len, chunk, encoding, cb); - entry = entry.next; - // if we didn't call the onwrite immediately, then - // it means that we need to wait until it does. - // also, that means that the chunk and cb are currently - // being processed, so move the buffer counter past them. - if (state.writing) { - break; - } - } - - if (entry === null) state.lastBufferedRequest = null; - } - - state.bufferedRequestCount = 0; - state.bufferedRequest = entry; - state.bufferProcessing = false; - } - - Writable.prototype._write = function (chunk, encoding, cb) { - cb(new Error('not implemented')); - }; - - Writable.prototype._writev = null; - - Writable.prototype.end = function (chunk, encoding, cb) { - var state = this._writableState; - - if (typeof chunk === 'function') { - cb = chunk; - chunk = null; - encoding = null; - } else if (typeof encoding === 'function') { - cb = encoding; - encoding = null; - } - - if (chunk !== null && chunk !== undefined) this.write(chunk, encoding); - - // .end() fully uncorks - if (state.corked) { - state.corked = 1; - this.uncork(); - } - - // ignore unnecessary end() calls. - if (!state.ending && !state.finished) endWritable(this, state, cb); - }; - - function needFinish(state) { - return state.ending && state.length === 0 && state.bufferedRequest === null && !state.finished && !state.writing; - } - - function prefinish(stream, state) { - if (!state.prefinished) { - state.prefinished = true; - stream.emit('prefinish'); - } - } - - function finishMaybe(stream, state) { - var need = needFinish(state); - if (need) { - if (state.pendingcb === 0) { - prefinish(stream, state); - state.finished = true; - stream.emit('finish'); - } else { - prefinish(stream, state); - } - } - return need; - } - - function endWritable(stream, state, cb) { - state.ending = true; - finishMaybe(stream, state); - if (cb) { - if (state.finished) nextTick(cb);else stream.once('finish', cb); - } - state.ended = true; - stream.writable = false; - } - - // It seems a linked list but it is not - // there will be only 2 of these for each stream - function CorkedRequest(state) { - var _this = this; - - this.next = null; - this.entry = null; - - this.finish = function (err) { - var entry = _this.entry; - _this.entry = null; - while (entry) { - var cb = entry.callback; - state.pendingcb--; - cb(err); - entry = entry.next; - } - if (state.corkedRequestsFree) { - state.corkedRequestsFree.next = _this; - } else { - state.corkedRequestsFree = _this; - } - }; - } - - inherits$1(Duplex, Readable); - - var keys = Object.keys(Writable.prototype); - for (var v = 0; v < keys.length; v++) { - var method = keys[v]; - if (!Duplex.prototype[method]) Duplex.prototype[method] = Writable.prototype[method]; - } - function Duplex(options) { - if (!(this instanceof Duplex)) return new Duplex(options); - - Readable.call(this, options); - Writable.call(this, options); - - if (options && options.readable === false) this.readable = false; - - if (options && options.writable === false) this.writable = false; - - this.allowHalfOpen = true; - if (options && options.allowHalfOpen === false) this.allowHalfOpen = false; - - this.once('end', onend); - } - - // the no-half-open enforcer - function onend() { - // if we allow half-open state, or if the writable side ended, - // then we're ok. - if (this.allowHalfOpen || this._writableState.ended) return; - - // no more data can be written. - // But allow more writes to happen in this tick. - nextTick(onEndNT, this); - } - - function onEndNT(self) { - self.end(); - } - - // a transform stream is a readable/writable stream where you do - inherits$1(Transform, Duplex); - - function TransformState(stream) { - this.afterTransform = function (er, data) { - return afterTransform(stream, er, data); - }; - - this.needTransform = false; - this.transforming = false; - this.writecb = null; - this.writechunk = null; - this.writeencoding = null; - } - - function afterTransform(stream, er, data) { - var ts = stream._transformState; - ts.transforming = false; - - var cb = ts.writecb; - - if (!cb) return stream.emit('error', new Error('no writecb in Transform class')); - - ts.writechunk = null; - ts.writecb = null; - - if (data !== null && data !== undefined) stream.push(data); - - cb(er); - - var rs = stream._readableState; - rs.reading = false; - if (rs.needReadable || rs.length < rs.highWaterMark) { - stream._read(rs.highWaterMark); - } - } - function Transform(options) { - if (!(this instanceof Transform)) return new Transform(options); - - Duplex.call(this, options); - - this._transformState = new TransformState(this); - - // when the writable side finishes, then flush out anything remaining. - var stream = this; - - // start out asking for a readable event once data is transformed. - this._readableState.needReadable = true; - - // we have implemented the _read method, and done the other things - // that Readable wants before the first _read call, so unset the - // sync guard flag. - this._readableState.sync = false; - - if (options) { - if (typeof options.transform === 'function') this._transform = options.transform; - - if (typeof options.flush === 'function') this._flush = options.flush; - } - - this.once('prefinish', function () { - if (typeof this._flush === 'function') this._flush(function (er) { - done(stream, er); - });else done(stream); - }); - } - - Transform.prototype.push = function (chunk, encoding) { - this._transformState.needTransform = false; - return Duplex.prototype.push.call(this, chunk, encoding); - }; - - // This is the part where you do stuff! - // override this function in implementation classes. - // 'chunk' is an input chunk. - // - // Call `push(newChunk)` to pass along transformed output - // to the readable side. You may call 'push' zero or more times. - // - // Call `cb(err)` when you are done with this chunk. If you pass - // an error, then that'll put the hurt on the whole operation. If you - // never call cb(), then you'll never get another chunk. - Transform.prototype._transform = function (chunk, encoding, cb) { - throw new Error('Not implemented'); - }; - - Transform.prototype._write = function (chunk, encoding, cb) { - var ts = this._transformState; - ts.writecb = cb; - ts.writechunk = chunk; - ts.writeencoding = encoding; - if (!ts.transforming) { - var rs = this._readableState; - if (ts.needTransform || rs.needReadable || rs.length < rs.highWaterMark) this._read(rs.highWaterMark); - } - }; - - // Doesn't matter what the args are here. - // _transform does all the work. - // That we got here means that the readable side wants more data. - Transform.prototype._read = function (n) { - var ts = this._transformState; - - if (ts.writechunk !== null && ts.writecb && !ts.transforming) { - ts.transforming = true; - this._transform(ts.writechunk, ts.writeencoding, ts.afterTransform); - } else { - // mark that we need a transform, so that any data that comes in - // will get processed, now that we've asked for it. - ts.needTransform = true; - } - }; - - function done(stream, er) { - if (er) return stream.emit('error', er); - - // if there's nothing in the write buffer, then that means - // that nothing more will ever be provided - var ws = stream._writableState; - var ts = stream._transformState; - - if (ws.length) throw new Error('Calling transform done when ws.length != 0'); - - if (ts.transforming) throw new Error('Calling transform done when still transforming'); - - return stream.push(null); - } - - inherits$1(PassThrough, Transform); - function PassThrough(options) { - if (!(this instanceof PassThrough)) return new PassThrough(options); - - Transform.call(this, options); - } - - PassThrough.prototype._transform = function (chunk, encoding, cb) { - cb(null, chunk); - }; - - inherits$1(Stream, EventEmitter); - Stream.Readable = Readable; - Stream.Writable = Writable; - Stream.Duplex = Duplex; - Stream.Transform = Transform; - Stream.PassThrough = PassThrough; - - // Backwards-compat with node 0.4.x - Stream.Stream = Stream; - - // old-style streams. Note that the pipe method (the only relevant - // part of this class) is overridden in the Readable class. - - function Stream() { - EventEmitter.call(this); - } - - Stream.prototype.pipe = function(dest, options) { - var source = this; - - function ondata(chunk) { - if (dest.writable) { - if (false === dest.write(chunk) && source.pause) { - source.pause(); - } - } - } - - source.on('data', ondata); - - function ondrain() { - if (source.readable && source.resume) { - source.resume(); - } - } - - dest.on('drain', ondrain); - - // If the 'end' option is not supplied, dest.end() will be called when - // source gets the 'end' or 'close' events. Only dest.end() once. - if (!dest._isStdio && (!options || options.end !== false)) { - source.on('end', onend); - source.on('close', onclose); - } - - var didOnEnd = false; - function onend() { - if (didOnEnd) return; - didOnEnd = true; - - dest.end(); - } - - - function onclose() { - if (didOnEnd) return; - didOnEnd = true; - - if (typeof dest.destroy === 'function') dest.destroy(); - } - - // don't leave dangling pipes when there are errors. - function onerror(er) { - cleanup(); - if (EventEmitter.listenerCount(this, 'error') === 0) { - throw er; // Unhandled stream error in pipe. - } - } - - source.on('error', onerror); - dest.on('error', onerror); - - // remove all the event listeners that were added. - function cleanup() { - source.removeListener('data', ondata); - dest.removeListener('drain', ondrain); - - source.removeListener('end', onend); - source.removeListener('close', onclose); - - source.removeListener('error', onerror); - dest.removeListener('error', onerror); - - source.removeListener('end', cleanup); - source.removeListener('close', cleanup); - - dest.removeListener('close', cleanup); - } - - source.on('end', cleanup); - source.on('close', cleanup); - - dest.on('close', cleanup); - - dest.emit('pipe', source); - - // Allow for unix-like usage: A.pipe(B).pipe(C) - return dest; - }; - - const bom_utf8 = Buffer.from([239, 187, 191]); - - class CsvError extends Error { - constructor(code, message, ...contexts) { - if(Array.isArray(message)) message = message.join(' '); - super(message); - if(Error.captureStackTrace !== undefined){ - Error.captureStackTrace(this, CsvError); - } - this.code = code; - for(const context of contexts){ - for(const key in context){ - const value = context[key]; - this[key] = isBuffer(value) ? value.toString() : value == null ? value : JSON.parse(JSON.stringify(value)); - } - } - } - } - - const isObject = function(obj){ - return typeof obj === 'object' && obj !== null && ! Array.isArray(obj); - }; - - const underscore = function(str){ - return str.replace(/([A-Z])/g, function(_, match){ - return '_' + match.toLowerCase(); - }); - }; - // Lodash implementation of `get` const charCodeOfDot = '.'.charCodeAt(0); @@ -5078,455 +2046,473 @@ return (index && index === length) ? object : undefined; }; - class Stringifier extends Transform { - constructor(opts = {}){ - super({...{writableObjectMode: true}, ...opts}); - const options = {}; - let err; - // Merge with user options - for(const opt in opts){ - options[underscore(opt)] = opts[opt]; - } - if((err = this.normalize(options)) !== undefined) throw err; - switch(options.record_delimiter){ - case 'auto': - options.record_delimiter = null; - break; - case 'unix': - options.record_delimiter = "\n"; - break; - case 'mac': - options.record_delimiter = "\r"; - break; - case 'windows': - options.record_delimiter = "\r\n"; - break; - case 'ascii': - options.record_delimiter = "\u001e"; - break; - case 'unicode': - options.record_delimiter = "\u2028"; - break; - } - // Expose options - this.options = options; - // Internal state - this.state = { - stop: false - }; - // Information - this.info = { - records: 0 - }; - } - normalize(options){ - // Normalize option `bom` - if(options.bom === undefined || options.bom === null || options.bom === false){ - options.bom = false; - }else if(options.bom !== true){ - return new CsvError('CSV_OPTION_BOOLEAN_INVALID_TYPE', [ - 'option `bom` is optional and must be a boolean value,', - `got ${JSON.stringify(options.bom)}` - ]); - } - // Normalize option `delimiter` - if(options.delimiter === undefined || options.delimiter === null){ - options.delimiter = ','; - }else if(isBuffer(options.delimiter)){ - options.delimiter = options.delimiter.toString(); - }else if(typeof options.delimiter !== 'string'){ - return new CsvError('CSV_OPTION_DELIMITER_INVALID_TYPE', [ - 'option `delimiter` must be a buffer or a string,', - `got ${JSON.stringify(options.delimiter)}` - ]); - } - // Normalize option `quote` - if(options.quote === undefined || options.quote === null){ - options.quote = '"'; - }else if(options.quote === true){ - options.quote = '"'; - }else if(options.quote === false){ - options.quote = ''; - }else if (isBuffer(options.quote)){ - options.quote = options.quote.toString(); - }else if(typeof options.quote !== 'string'){ - return new CsvError('CSV_OPTION_QUOTE_INVALID_TYPE', [ - 'option `quote` must be a boolean, a buffer or a string,', - `got ${JSON.stringify(options.quote)}` - ]); - } - // Normalize option `quoted` - if(options.quoted === undefined || options.quoted === null){ - options.quoted = false; - } - // Normalize option `quoted_empty` - if(options.quoted_empty === undefined || options.quoted_empty === null){ - options.quoted_empty = undefined; - } - // Normalize option `quoted_match` - if(options.quoted_match === undefined || options.quoted_match === null || options.quoted_match === false){ - options.quoted_match = null; - }else if(!Array.isArray(options.quoted_match)){ - options.quoted_match = [options.quoted_match]; + const is_object = function(obj){ + return typeof obj === 'object' && obj !== null && ! Array.isArray(obj); + }; + + const normalize_columns = function(columns){ + if(columns === undefined || columns === null){ + return [undefined, undefined]; + } + if(typeof columns !== 'object'){ + return [Error('Invalid option "columns": expect an array or an object')]; + } + if(!Array.isArray(columns)){ + const newcolumns = []; + for(const k in columns){ + newcolumns.push({ + key: k, + header: columns[k] + }); } - if(options.quoted_match){ - for(const quoted_match of options.quoted_match){ - const isString = typeof quoted_match === 'string'; - const isRegExp = quoted_match instanceof RegExp; - if(!isString && !isRegExp){ - return Error(`Invalid Option: quoted_match must be a string or a regex, got ${JSON.stringify(quoted_match)}`); + columns = newcolumns; + }else { + const newcolumns = []; + for(const column of columns){ + if(typeof column === 'string'){ + newcolumns.push({ + key: column, + header: column + }); + }else if(typeof column === 'object' && column !== null && !Array.isArray(column)){ + if(!column.key){ + return [Error('Invalid column definition: property "key" is required')]; + } + if(column.header === undefined){ + column.header = column.key; } + newcolumns.push(column); + }else { + return [Error('Invalid column definition: expect a string or an object')]; } } - // Normalize option `quoted_string` - if(options.quoted_string === undefined || options.quoted_string === null){ - options.quoted_string = false; - } - // Normalize option `eof` - if(options.eof === undefined || options.eof === null){ - options.eof = true; - } - // Normalize option `escape` - if(options.escape === undefined || options.escape === null){ - options.escape = '"'; - }else if(isBuffer(options.escape)){ - options.escape = options.escape.toString(); - }else if(typeof options.escape !== 'string'){ - return Error(`Invalid Option: escape must be a buffer or a string, got ${JSON.stringify(options.escape)}`); - } - if (options.escape.length > 1){ - return Error(`Invalid Option: escape must be one character, got ${options.escape.length} characters`); - } - // Normalize option `header` - if(options.header === undefined || options.header === null){ - options.header = false; - } - // Normalize option `columns` - const [err, columns] = this.normalize_columns(options.columns); - if(err) return err; - options.columns = columns; - // Normalize option `quoted` - if(options.quoted === undefined || options.quoted === null){ - options.quoted = false; - } - // Normalize option `cast` - if(options.cast === undefined || options.cast === null){ - options.cast = {}; - } - // Normalize option cast.bigint - if(options.cast.bigint === undefined || options.cast.bigint === null){ - // Cast boolean to string by default - options.cast.bigint = value => '' + value; - } - // Normalize option cast.boolean - if(options.cast.boolean === undefined || options.cast.boolean === null){ - // Cast boolean to string by default - options.cast.boolean = value => value ? '1' : ''; - } - // Normalize option cast.date - if(options.cast.date === undefined || options.cast.date === null){ - // Cast date to timestamp string by default - options.cast.date = value => '' + value.getTime(); - } - // Normalize option cast.number - if(options.cast.number === undefined || options.cast.number === null){ - // Cast number to string using native casting by default - options.cast.number = value => '' + value; - } - // Normalize option cast.object - if(options.cast.object === undefined || options.cast.object === null){ - // Stringify object as JSON by default - options.cast.object = value => JSON.stringify(value); - } - // Normalize option cast.string - if(options.cast.string === undefined || options.cast.string === null){ - // Leave string untouched - options.cast.string = function(value){return value;}; - } - // Normalize option `record_delimiter` - if(options.record_delimiter === undefined || options.record_delimiter === null){ - options.record_delimiter = '\n'; - }else if(isBuffer(options.record_delimiter)){ - options.record_delimiter = options.record_delimiter.toString(); - }else if(typeof options.record_delimiter !== 'string'){ - return Error(`Invalid Option: record_delimiter must be a buffer or a string, got ${JSON.stringify(options.record_delimiter)}`); - } + columns = newcolumns; } - _transform(chunk, encoding, callback){ - if(this.state.stop === true){ - return; + return [undefined, columns]; + }; + + class CsvError extends Error { + constructor(code, message, ...contexts) { + if(Array.isArray(message)) message = message.join(' '); + super(message); + if(Error.captureStackTrace !== undefined){ + Error.captureStackTrace(this, CsvError); } - const err = this.__transform(chunk); - if(err !== undefined){ - this.state.stop = true; + this.code = code; + for(const context of contexts){ + for(const key in context){ + const value = context[key]; + this[key] = isBuffer(value) ? value.toString() : value == null ? value : JSON.parse(JSON.stringify(value)); + } } - callback(err); } - _flush(callback){ - if(this.state.stop === true){ - // Note, Node.js 12 call flush even after an error, we must prevent - // `callback` from being called in flush without any error. - return; - } - if(this.info.records === 0){ - this.bom(); - const err = this.headers(); - if(err) callback(err); + } + + const underscore = function(str){ + return str.replace(/([A-Z])/g, function(_, match){ + return '_' + match.toLowerCase(); + }); + }; + + const normalize_options = function(opts) { + const options = {}; + // Merge with user options + for(const opt in opts){ + options[underscore(opt)] = opts[opt]; + } + // Normalize option `bom` + if(options.bom === undefined || options.bom === null || options.bom === false){ + options.bom = false; + }else if(options.bom !== true){ + return [new CsvError('CSV_OPTION_BOOLEAN_INVALID_TYPE', [ + 'option `bom` is optional and must be a boolean value,', + `got ${JSON.stringify(options.bom)}` + ])]; + } + // Normalize option `delimiter` + if(options.delimiter === undefined || options.delimiter === null){ + options.delimiter = ','; + }else if(isBuffer(options.delimiter)){ + options.delimiter = options.delimiter.toString(); + }else if(typeof options.delimiter !== 'string'){ + return [new CsvError('CSV_OPTION_DELIMITER_INVALID_TYPE', [ + 'option `delimiter` must be a buffer or a string,', + `got ${JSON.stringify(options.delimiter)}` + ])]; + } + // Normalize option `quote` + if(options.quote === undefined || options.quote === null){ + options.quote = '"'; + }else if(options.quote === true){ + options.quote = '"'; + }else if(options.quote === false){ + options.quote = ''; + }else if (isBuffer(options.quote)){ + options.quote = options.quote.toString(); + }else if(typeof options.quote !== 'string'){ + return [new CsvError('CSV_OPTION_QUOTE_INVALID_TYPE', [ + 'option `quote` must be a boolean, a buffer or a string,', + `got ${JSON.stringify(options.quote)}` + ])]; + } + // Normalize option `quoted` + if(options.quoted === undefined || options.quoted === null){ + options.quoted = false; + } + // Normalize option `quoted_empty` + if(options.quoted_empty === undefined || options.quoted_empty === null){ + options.quoted_empty = undefined; + } + // Normalize option `quoted_match` + if(options.quoted_match === undefined || options.quoted_match === null || options.quoted_match === false){ + options.quoted_match = null; + }else if(!Array.isArray(options.quoted_match)){ + options.quoted_match = [options.quoted_match]; + } + if(options.quoted_match){ + for(const quoted_match of options.quoted_match){ + const isString = typeof quoted_match === 'string'; + const isRegExp = quoted_match instanceof RegExp; + if(!isString && !isRegExp){ + return [Error(`Invalid Option: quoted_match must be a string or a regex, got ${JSON.stringify(quoted_match)}`)]; + } } - callback(); } - __transform(chunk){ - // Chunk validation - if(!Array.isArray(chunk) && typeof chunk !== 'object'){ - return Error(`Invalid Record: expect an array or an object, got ${JSON.stringify(chunk)}`); - } - // Detect columns from the first record - if(this.info.records === 0){ - if(Array.isArray(chunk)){ - if(this.options.header === true && this.options.columns === undefined){ - return Error('Undiscoverable Columns: header option requires column option or object records'); + // Normalize option `quoted_string` + if(options.quoted_string === undefined || options.quoted_string === null){ + options.quoted_string = false; + } + // Normalize option `eof` + if(options.eof === undefined || options.eof === null){ + options.eof = true; + } + // Normalize option `escape` + if(options.escape === undefined || options.escape === null){ + options.escape = '"'; + }else if(isBuffer(options.escape)){ + options.escape = options.escape.toString(); + }else if(typeof options.escape !== 'string'){ + return [Error(`Invalid Option: escape must be a buffer or a string, got ${JSON.stringify(options.escape)}`)]; + } + if (options.escape.length > 1){ + return [Error(`Invalid Option: escape must be one character, got ${options.escape.length} characters`)]; + } + // Normalize option `header` + if(options.header === undefined || options.header === null){ + options.header = false; + } + // Normalize option `columns` + const [errColumns, columns] = normalize_columns(options.columns); + if(errColumns !== undefined) return [errColumns]; + options.columns = columns; + // Normalize option `quoted` + if(options.quoted === undefined || options.quoted === null){ + options.quoted = false; + } + // Normalize option `cast` + if(options.cast === undefined || options.cast === null){ + options.cast = {}; + } + // Normalize option cast.bigint + if(options.cast.bigint === undefined || options.cast.bigint === null){ + // Cast boolean to string by default + options.cast.bigint = value => '' + value; + } + // Normalize option cast.boolean + if(options.cast.boolean === undefined || options.cast.boolean === null){ + // Cast boolean to string by default + options.cast.boolean = value => value ? '1' : ''; + } + // Normalize option cast.date + if(options.cast.date === undefined || options.cast.date === null){ + // Cast date to timestamp string by default + options.cast.date = value => '' + value.getTime(); + } + // Normalize option cast.number + if(options.cast.number === undefined || options.cast.number === null){ + // Cast number to string using native casting by default + options.cast.number = value => '' + value; + } + // Normalize option cast.object + if(options.cast.object === undefined || options.cast.object === null){ + // Stringify object as JSON by default + options.cast.object = value => JSON.stringify(value); + } + // Normalize option cast.string + if(options.cast.string === undefined || options.cast.string === null){ + // Leave string untouched + options.cast.string = function(value){return value;}; + } + // Normalize option `on_record` + if(options.on_record !== undefined && typeof options.on_record !== 'function'){ + return [Error(`Invalid Option: "on_record" must be a function.`)]; + } + // Normalize option `record_delimiter` + if(options.record_delimiter === undefined || options.record_delimiter === null){ + options.record_delimiter = '\n'; + }else if(isBuffer(options.record_delimiter)){ + options.record_delimiter = options.record_delimiter.toString(); + }else if(typeof options.record_delimiter !== 'string'){ + return [Error(`Invalid Option: record_delimiter must be a buffer or a string, got ${JSON.stringify(options.record_delimiter)}`)]; + } + switch(options.record_delimiter){ + case 'auto': + options.record_delimiter = null; + break; + case 'unix': + options.record_delimiter = "\n"; + break; + case 'mac': + options.record_delimiter = "\r"; + break; + case 'windows': + options.record_delimiter = "\r\n"; + break; + case 'ascii': + options.record_delimiter = "\u001e"; + break; + case 'unicode': + options.record_delimiter = "\u2028"; + break; + } + return [undefined, options]; + }; + + const bom_utf8 = Buffer.from([239, 187, 191]); + + const stringifier = function(options, state, info){ + return { + options: options, + state: state, + info: info, + __transform: function(chunk, push){ + // Chunk validation + if(!Array.isArray(chunk) && typeof chunk !== 'object'){ + return Error(`Invalid Record: expect an array or an object, got ${JSON.stringify(chunk)}`); + } + // Detect columns from the first record + if(this.info.records === 0){ + if(Array.isArray(chunk)){ + if(this.options.header === true && this.options.columns === undefined){ + return Error('Undiscoverable Columns: header option requires column option or object records'); + } + }else if(this.options.columns === undefined){ + const [err, columns] = normalize_columns(Object.keys(chunk)); + if(err) return; + this.options.columns = columns; } - }else if(this.options.columns === undefined){ - const [err, columns] = this.normalize_columns(Object.keys(chunk)); - if(err) return; - this.options.columns = columns; } - } - // Emit the header - if(this.info.records === 0){ - this.bom(); - const err = this.headers(); - if(err) return err; - } - // Emit and stringify the record if an object or an array - try{ - this.emit('record', chunk, this.info.records); - }catch(err){ - return err; - } - // Convert the record into a string - let err, chunk_string; - if(this.options.eof){ - [err, chunk_string] = this.stringify(chunk); - if(err) return err; - if(chunk_string === undefined){ - return; - }else { - chunk_string = chunk_string + this.options.record_delimiter; + // Emit the header + if(this.info.records === 0){ + this.bom(push); + const err = this.headers(push); + if(err) return err; } - }else { - [err, chunk_string] = this.stringify(chunk); - if(err) return err; - if(chunk_string === undefined){ - return; - }else { - if(this.options.header || this.info.records){ - chunk_string = this.options.record_delimiter + chunk_string; + // Emit and stringify the record if an object or an array + try{ + // this.emit('record', chunk, this.info.records); + if(this.options.on_record){ + this.options.on_record(chunk, this.info.records); } + }catch(err){ + return err; } - } - // Emit the csv - this.info.records++; - this.push(chunk_string); - } - stringify(chunk, chunkIsHeader=false){ - if(typeof chunk !== 'object'){ - return [undefined, chunk]; - } - const {columns} = this.options; - const record = []; - // Record is an array - if(Array.isArray(chunk)){ - // We are getting an array but the user has specified output columns. In - // this case, we respect the columns indexes - if(columns){ - chunk.splice(columns.length); - } - // Cast record elements - for(let i=0; i= 0; - const containsQuote = (quote !== '') && value.indexOf(quote) >= 0; - const containsEscape = value.indexOf(escape) >= 0 && (escape !== quote); - const containsRecordDelimiter = value.indexOf(record_delimiter) >= 0; - const quotedString = quoted_string && typeof field === 'string'; - let quotedMatch = quoted_match && quoted_match.filter(quoted_match => { - if(typeof quoted_match === 'string'){ - return value.indexOf(quoted_match) !== -1; - }else { - return quoted_match.test(value); + } + let csvrecord = ''; + for(let i=0; i 0; - const shouldQuote = containsQuote === true || containsdelimiter || containsRecordDelimiter || quoted || quotedString || quotedMatch; - if(shouldQuote === true && containsEscape === true){ - const regexp = escape === '\\' - ? new RegExp(escape + escape, 'g') - : new RegExp(escape, 'g'); - value = value.replace(regexp, escape + escape); + options = {...this.options, ...options}; + [err, options] = normalize_options(options); + if(err !== undefined){ + return [err]; + } + }else if(value === undefined || value === null){ + options = this.options; + }else { + return [Error(`Invalid Casting Value: returned value must return a string, an object, null or undefined, got ${JSON.stringify(value)}`)]; } - if(containsQuote === true){ - const regexp = new RegExp(quote,'g'); - value = value.replace(regexp, escape + quote); + const {delimiter, escape, quote, quoted, quoted_empty, quoted_string, quoted_match, record_delimiter} = options; + if(value){ + if(typeof value !== 'string'){ + return [Error(`Formatter must return a string, null or undefined, got ${JSON.stringify(value)}`)]; + } + const containsdelimiter = delimiter.length && value.indexOf(delimiter) >= 0; + const containsQuote = (quote !== '') && value.indexOf(quote) >= 0; + const containsEscape = value.indexOf(escape) >= 0 && (escape !== quote); + const containsRecordDelimiter = value.indexOf(record_delimiter) >= 0; + const quotedString = quoted_string && typeof field === 'string'; + let quotedMatch = quoted_match && quoted_match.filter(quoted_match => { + if(typeof quoted_match === 'string'){ + return value.indexOf(quoted_match) !== -1; + }else { + return quoted_match.test(value); + } + }); + quotedMatch = quotedMatch && quotedMatch.length > 0; + const shouldQuote = containsQuote === true || containsdelimiter || containsRecordDelimiter || quoted || quotedString || quotedMatch; + if(shouldQuote === true && containsEscape === true){ + const regexp = escape === '\\' + ? new RegExp(escape + escape, 'g') + : new RegExp(escape, 'g'); + value = value.replace(regexp, escape + escape); + } + if(containsQuote === true){ + const regexp = new RegExp(quote,'g'); + value = value.replace(regexp, escape + quote); + } + if(shouldQuote === true){ + value = quote + value + quote; + } + csvrecord += value; + }else if(quoted_empty === true || (field === '' && quoted_string === true && quoted_empty !== false)){ + csvrecord += quote + quote; } - if(shouldQuote === true){ - value = quote + value + quote; + if(i !== record.length - 1){ + csvrecord += delimiter; } - csvrecord += value; - }else if(quoted_empty === true || (field === '' && quoted_string === true && quoted_empty !== false)){ - csvrecord += quote + quote; } - if(i !== record.length - 1){ - csvrecord += delimiter; + return [undefined, csvrecord]; + }, + bom: function(push){ + if(this.options.bom !== true){ + return; } - } - return [undefined, csvrecord]; - } - bom(){ - if(this.options.bom !== true){ - return; - } - this.push(bom_utf8); - } - headers(){ - if(this.options.header === false){ - return; - } - if(this.options.columns === undefined){ - return; - } - let err; - let headers = this.options.columns.map(column => column.header); - if(this.options.eof){ - [err, headers] = this.stringify(headers, true); - headers += this.options.record_delimiter; - }else { - [err, headers] = this.stringify(headers); - } - if(err) return err; - this.push(headers); - } - __cast(value, context){ - const type = typeof value; - try{ - if(type === 'string'){ // Fine for 99% of the cases - return [undefined, this.options.cast.string(value, context)]; - }else if(type === 'bigint'){ - return [undefined, this.options.cast.bigint(value, context)]; - }else if(type === 'number'){ - return [undefined, this.options.cast.number(value, context)]; - }else if(type === 'boolean'){ - return [undefined, this.options.cast.boolean(value, context)]; - }else if(value instanceof Date){ - return [undefined, this.options.cast.date(value, context)]; - }else if(type === 'object' && value !== null){ - return [undefined, this.options.cast.object(value, context)]; - }else { - return [undefined, value, value]; + push(bom_utf8); + }, + headers: function(push){ + if(this.options.header === false){ + return; } - }catch(err){ - return [err]; - } - } - normalize_columns(columns){ - if(columns === undefined || columns === null){ - return []; - } - if(typeof columns !== 'object'){ - return [Error('Invalid option "columns": expect an array or an object')]; - } - if(!Array.isArray(columns)){ - const newcolumns = []; - for(const k in columns){ - newcolumns.push({ - key: k, - header: columns[k] - }); + if(this.options.columns === undefined){ + return; } - columns = newcolumns; - }else { - const newcolumns = []; - for(const column of columns){ - if(typeof column === 'string'){ - newcolumns.push({ - key: column, - header: column - }); - }else if(typeof column === 'object' && column !== undefined && !Array.isArray(column)){ - if(!column.key){ - return [Error('Invalid column definition: property "key" is required')]; - } - if(column.header === undefined){ - column.header = column.key; - } - newcolumns.push(column); + let err; + let headers = this.options.columns.map(column => column.header); + if(this.options.eof){ + [err, headers] = this.stringify(headers, true); + headers += this.options.record_delimiter; + }else { + [err, headers] = this.stringify(headers); + } + if(err) return err; + push(headers); + }, + __cast: function(value, context){ + const type = typeof value; + try{ + if(type === 'string'){ // Fine for 99% of the cases + return [undefined, this.options.cast.string(value, context)]; + }else if(type === 'bigint'){ + return [undefined, this.options.cast.bigint(value, context)]; + }else if(type === 'number'){ + return [undefined, this.options.cast.number(value, context)]; + }else if(type === 'boolean'){ + return [undefined, this.options.cast.boolean(value, context)]; + }else if(value instanceof Date){ + return [undefined, this.options.cast.date(value, context)]; + }else if(type === 'object' && value !== null){ + return [undefined, this.options.cast.object(value, context)]; }else { - return [Error('Invalid column definition: expect a string or an object')]; + return [undefined, value, value]; } + }catch(err){ + return [err]; } - columns = newcolumns; } - return [undefined, columns]; - } - } + }; + }; - const stringify = function(records, options={}){ + const stringify = function(records, opts={}){ const data = []; - const stringifier = new Stringifier(options); - stringifier.push = function(record){ - if(record === null){ - return; - } - data.push(record.toString()); + const [err, options] = normalize_options(opts); + if(err !== undefined) throw err; + const state = { + stop: false }; + // Information + const info = { + records: 0 + }; + const api = stringifier(options, state, info); + // stringifier.push = function(record){ + // if(record === null){ + // return; + // } + // data.push(record.toString()); + // }; for(const record of records){ - const err = stringifier.__transform(record, null); + const err = api.__transform(record, function(record){ + data.push(record); + }); if(err !== undefined) throw err; } return data.join(''); diff --git a/packages/csv-stringify/lib/api/CsvError.js b/packages/csv-stringify/lib/api/CsvError.js new file mode 100644 index 000000000..5725c7d27 --- /dev/null +++ b/packages/csv-stringify/lib/api/CsvError.js @@ -0,0 +1,19 @@ + +class CsvError extends Error { + constructor(code, message, ...contexts) { + if(Array.isArray(message)) message = message.join(' '); + super(message); + if(Error.captureStackTrace !== undefined){ + Error.captureStackTrace(this, CsvError); + } + this.code = code; + for(const context of contexts){ + for(const key in context){ + const value = context[key]; + this[key] = Buffer.isBuffer(value) ? value.toString() : value == null ? value : JSON.parse(JSON.stringify(value)); + } + } + } +} + +export { CsvError }; diff --git a/packages/csv-stringify/lib/api/index.js b/packages/csv-stringify/lib/api/index.js new file mode 100644 index 000000000..5b785875e --- /dev/null +++ b/packages/csv-stringify/lib/api/index.js @@ -0,0 +1,219 @@ + +import { get } from '../utils/get.js'; +import { is_object } from '../utils/is_object.js'; +import { normalize_columns } from './normalize_columns.js'; +import { normalize_options } from './normalize_options.js'; +const bom_utf8 = Buffer.from([239, 187, 191]); + +const stringifier = function(options, state, info){ + return { + options: options, + state: state, + info: info, + __transform: function(chunk, push){ + // Chunk validation + if(!Array.isArray(chunk) && typeof chunk !== 'object'){ + return Error(`Invalid Record: expect an array or an object, got ${JSON.stringify(chunk)}`); + } + // Detect columns from the first record + if(this.info.records === 0){ + if(Array.isArray(chunk)){ + if(this.options.header === true && this.options.columns === undefined){ + return Error('Undiscoverable Columns: header option requires column option or object records'); + } + }else if(this.options.columns === undefined){ + const [err, columns] = normalize_columns(Object.keys(chunk)); + if(err) return; + this.options.columns = columns; + } + } + // Emit the header + if(this.info.records === 0){ + this.bom(push); + const err = this.headers(push); + if(err) return err; + } + // Emit and stringify the record if an object or an array + try{ + // this.emit('record', chunk, this.info.records); + if(this.options.on_record){ + this.options.on_record(chunk, this.info.records); + } + }catch(err){ + return err; + } + // Convert the record into a string + let err, chunk_string; + if(this.options.eof){ + [err, chunk_string] = this.stringify(chunk); + if(err) return err; + if(chunk_string === undefined){ + return; + }else{ + chunk_string = chunk_string + this.options.record_delimiter; + } + }else{ + [err, chunk_string] = this.stringify(chunk); + if(err) return err; + if(chunk_string === undefined){ + return; + }else{ + if(this.options.header || this.info.records){ + chunk_string = this.options.record_delimiter + chunk_string; + } + } + } + // Emit the csv + this.info.records++; + push(chunk_string); + }, + stringify: function(chunk, chunkIsHeader=false){ + if(typeof chunk !== 'object'){ + return [undefined, chunk]; + } + const {columns} = this.options; + const record = []; + // Record is an array + if(Array.isArray(chunk)){ + // We are getting an array but the user has specified output columns. In + // this case, we respect the columns indexes + if(columns){ + chunk.splice(columns.length); + } + // Cast record elements + for(let i=0; i= 0; + const containsQuote = (quote !== '') && value.indexOf(quote) >= 0; + const containsEscape = value.indexOf(escape) >= 0 && (escape !== quote); + const containsRecordDelimiter = value.indexOf(record_delimiter) >= 0; + const quotedString = quoted_string && typeof field === 'string'; + let quotedMatch = quoted_match && quoted_match.filter(quoted_match => { + if(typeof quoted_match === 'string'){ + return value.indexOf(quoted_match) !== -1; + }else{ + return quoted_match.test(value); + } + }); + quotedMatch = quotedMatch && quotedMatch.length > 0; + const shouldQuote = containsQuote === true || containsdelimiter || containsRecordDelimiter || quoted || quotedString || quotedMatch; + if(shouldQuote === true && containsEscape === true){ + const regexp = escape === '\\' + ? new RegExp(escape + escape, 'g') + : new RegExp(escape, 'g'); + value = value.replace(regexp, escape + escape); + } + if(containsQuote === true){ + const regexp = new RegExp(quote,'g'); + value = value.replace(regexp, escape + quote); + } + if(shouldQuote === true){ + value = quote + value + quote; + } + csvrecord += value; + }else if(quoted_empty === true || (field === '' && quoted_string === true && quoted_empty !== false)){ + csvrecord += quote + quote; + } + if(i !== record.length - 1){ + csvrecord += delimiter; + } + } + return [undefined, csvrecord]; + }, + bom: function(push){ + if(this.options.bom !== true){ + return; + } + push(bom_utf8); + }, + headers: function(push){ + if(this.options.header === false){ + return; + } + if(this.options.columns === undefined){ + return; + } + let err; + let headers = this.options.columns.map(column => column.header); + if(this.options.eof){ + [err, headers] = this.stringify(headers, true); + headers += this.options.record_delimiter; + }else{ + [err, headers] = this.stringify(headers); + } + if(err) return err; + push(headers); + }, + __cast: function(value, context){ + const type = typeof value; + try{ + if(type === 'string'){ // Fine for 99% of the cases + return [undefined, this.options.cast.string(value, context)]; + }else if(type === 'bigint'){ + return [undefined, this.options.cast.bigint(value, context)]; + }else if(type === 'number'){ + return [undefined, this.options.cast.number(value, context)]; + }else if(type === 'boolean'){ + return [undefined, this.options.cast.boolean(value, context)]; + }else if(value instanceof Date){ + return [undefined, this.options.cast.date(value, context)]; + }else if(type === 'object' && value !== null){ + return [undefined, this.options.cast.object(value, context)]; + }else{ + return [undefined, value, value]; + } + }catch(err){ + return [err]; + } + } + }; +}; + +export {stringifier}; diff --git a/packages/csv-stringify/lib/api/normalize_columns.js b/packages/csv-stringify/lib/api/normalize_columns.js new file mode 100644 index 000000000..f9a177e66 --- /dev/null +++ b/packages/csv-stringify/lib/api/normalize_columns.js @@ -0,0 +1,43 @@ + +const normalize_columns = function(columns){ + if(columns === undefined || columns === null){ + return [undefined, undefined]; + } + if(typeof columns !== 'object'){ + return [Error('Invalid option "columns": expect an array or an object')]; + } + if(!Array.isArray(columns)){ + const newcolumns = []; + for(const k in columns){ + newcolumns.push({ + key: k, + header: columns[k] + }); + } + columns = newcolumns; + }else{ + const newcolumns = []; + for(const column of columns){ + if(typeof column === 'string'){ + newcolumns.push({ + key: column, + header: column + }); + }else if(typeof column === 'object' && column !== null && !Array.isArray(column)){ + if(!column.key){ + return [Error('Invalid column definition: property "key" is required')]; + } + if(column.header === undefined){ + column.header = column.key; + } + newcolumns.push(column); + }else{ + return [Error('Invalid column definition: expect a string or an object')]; + } + } + columns = newcolumns; + } + return [undefined, columns]; +}; + +export {normalize_columns}; diff --git a/packages/csv-stringify/lib/api/normalize_options.js b/packages/csv-stringify/lib/api/normalize_options.js new file mode 100644 index 000000000..7dd3450b0 --- /dev/null +++ b/packages/csv-stringify/lib/api/normalize_options.js @@ -0,0 +1,184 @@ + +import { CsvError } from './CsvError.js'; +import { normalize_columns } from './normalize_columns.js'; +import { underscore } from '../utils/underscore.js'; + +const normalize_options = function(opts) { + const options = {}; + // Merge with user options + for(const opt in opts){ + options[underscore(opt)] = opts[opt]; + } + // Normalize option `bom` + if(options.bom === undefined || options.bom === null || options.bom === false){ + options.bom = false; + }else if(options.bom !== true){ + return [new CsvError('CSV_OPTION_BOOLEAN_INVALID_TYPE', [ + 'option `bom` is optional and must be a boolean value,', + `got ${JSON.stringify(options.bom)}` + ])]; + } + // Normalize option `delimiter` + if(options.delimiter === undefined || options.delimiter === null){ + options.delimiter = ','; + }else if(Buffer.isBuffer(options.delimiter)){ + options.delimiter = options.delimiter.toString(); + }else if(typeof options.delimiter !== 'string'){ + return [new CsvError('CSV_OPTION_DELIMITER_INVALID_TYPE', [ + 'option `delimiter` must be a buffer or a string,', + `got ${JSON.stringify(options.delimiter)}` + ])]; + } + // Normalize option `quote` + if(options.quote === undefined || options.quote === null){ + options.quote = '"'; + }else if(options.quote === true){ + options.quote = '"'; + }else if(options.quote === false){ + options.quote = ''; + }else if (Buffer.isBuffer(options.quote)){ + options.quote = options.quote.toString(); + }else if(typeof options.quote !== 'string'){ + return [new CsvError('CSV_OPTION_QUOTE_INVALID_TYPE', [ + 'option `quote` must be a boolean, a buffer or a string,', + `got ${JSON.stringify(options.quote)}` + ])]; + } + // Normalize option `quoted` + if(options.quoted === undefined || options.quoted === null){ + options.quoted = false; + }else{ + // todo + } + // Normalize option `quoted_empty` + if(options.quoted_empty === undefined || options.quoted_empty === null){ + options.quoted_empty = undefined; + }else{ + // todo + } + // Normalize option `quoted_match` + if(options.quoted_match === undefined || options.quoted_match === null || options.quoted_match === false){ + options.quoted_match = null; + }else if(!Array.isArray(options.quoted_match)){ + options.quoted_match = [options.quoted_match]; + } + if(options.quoted_match){ + for(const quoted_match of options.quoted_match){ + const isString = typeof quoted_match === 'string'; + const isRegExp = quoted_match instanceof RegExp; + if(!isString && !isRegExp){ + return [Error(`Invalid Option: quoted_match must be a string or a regex, got ${JSON.stringify(quoted_match)}`)]; + } + } + } + // Normalize option `quoted_string` + if(options.quoted_string === undefined || options.quoted_string === null){ + options.quoted_string = false; + }else{ + // todo + } + // Normalize option `eof` + if(options.eof === undefined || options.eof === null){ + options.eof = true; + }else{ + // todo + } + // Normalize option `escape` + if(options.escape === undefined || options.escape === null){ + options.escape = '"'; + }else if(Buffer.isBuffer(options.escape)){ + options.escape = options.escape.toString(); + }else if(typeof options.escape !== 'string'){ + return [Error(`Invalid Option: escape must be a buffer or a string, got ${JSON.stringify(options.escape)}`)]; + } + if (options.escape.length > 1){ + return [Error(`Invalid Option: escape must be one character, got ${options.escape.length} characters`)]; + } + // Normalize option `header` + if(options.header === undefined || options.header === null){ + options.header = false; + }else{ + // todo + } + // Normalize option `columns` + const [errColumns, columns] = normalize_columns(options.columns); + if(errColumns !== undefined) return [errColumns]; + options.columns = columns; + // Normalize option `quoted` + if(options.quoted === undefined || options.quoted === null){ + options.quoted = false; + }else{ + // todo + } + // Normalize option `cast` + if(options.cast === undefined || options.cast === null){ + options.cast = {}; + }else{ + // todo + } + // Normalize option cast.bigint + if(options.cast.bigint === undefined || options.cast.bigint === null){ + // Cast boolean to string by default + options.cast.bigint = value => '' + value; + } + // Normalize option cast.boolean + if(options.cast.boolean === undefined || options.cast.boolean === null){ + // Cast boolean to string by default + options.cast.boolean = value => value ? '1' : ''; + } + // Normalize option cast.date + if(options.cast.date === undefined || options.cast.date === null){ + // Cast date to timestamp string by default + options.cast.date = value => '' + value.getTime(); + } + // Normalize option cast.number + if(options.cast.number === undefined || options.cast.number === null){ + // Cast number to string using native casting by default + options.cast.number = value => '' + value; + } + // Normalize option cast.object + if(options.cast.object === undefined || options.cast.object === null){ + // Stringify object as JSON by default + options.cast.object = value => JSON.stringify(value); + } + // Normalize option cast.string + if(options.cast.string === undefined || options.cast.string === null){ + // Leave string untouched + options.cast.string = function(value){return value;}; + } + // Normalize option `on_record` + if(options.on_record !== undefined && typeof options.on_record !== 'function'){ + return [Error(`Invalid Option: "on_record" must be a function.`)]; + } + // Normalize option `record_delimiter` + if(options.record_delimiter === undefined || options.record_delimiter === null){ + options.record_delimiter = '\n'; + }else if(Buffer.isBuffer(options.record_delimiter)){ + options.record_delimiter = options.record_delimiter.toString(); + }else if(typeof options.record_delimiter !== 'string'){ + return [Error(`Invalid Option: record_delimiter must be a buffer or a string, got ${JSON.stringify(options.record_delimiter)}`)]; + } + switch(options.record_delimiter){ + case 'auto': + options.record_delimiter = null; + break; + case 'unix': + options.record_delimiter = "\n"; + break; + case 'mac': + options.record_delimiter = "\r"; + break; + case 'windows': + options.record_delimiter = "\r\n"; + break; + case 'ascii': + options.record_delimiter = "\u001e"; + break; + case 'unicode': + options.record_delimiter = "\u2028"; + break; + } + return [undefined, options]; +}; + +export {normalize_options}; diff --git a/packages/csv-stringify/lib/index.js b/packages/csv-stringify/lib/index.js index 175049843..d6546a5f4 100644 --- a/packages/csv-stringify/lib/index.js +++ b/packages/csv-stringify/lib/index.js @@ -7,144 +7,16 @@ additional information. */ import { Transform } from 'stream'; -const bom_utf8 = Buffer.from([239, 187, 191]); - -class CsvError extends Error { - constructor(code, message, ...contexts) { - if(Array.isArray(message)) message = message.join(' '); - super(message); - if(Error.captureStackTrace !== undefined){ - Error.captureStackTrace(this, CsvError); - } - this.code = code; - for(const context of contexts){ - for(const key in context){ - const value = context[key]; - this[key] = Buffer.isBuffer(value) ? value.toString() : value == null ? value : JSON.parse(JSON.stringify(value)); - } - } - } -} - -const isObject = function(obj){ - return typeof obj === 'object' && obj !== null && ! Array.isArray(obj); -}; - -const underscore = function(str){ - return str.replace(/([A-Z])/g, function(_, match){ - return '_' + match.toLowerCase(); - }); -}; - -// Lodash implementation of `get` - -const charCodeOfDot = '.'.charCodeAt(0); -const reEscapeChar = /\\(\\)?/g; -const rePropName = RegExp( - // Match anything that isn't a dot or bracket. - '[^.[\\]]+' + '|' + - // Or match property names within brackets. - '\\[(?:' + - // Match a non-string expression. - '([^"\'][^[]*)' + '|' + - // Or match strings (supports escaping characters). - '(["\'])((?:(?!\\2)[^\\\\]|\\\\.)*?)\\2' + - ')\\]'+ '|' + - // Or match "" as the space between consecutive dots or empty brackets. - '(?=(?:\\.|\\[\\])(?:\\.|\\[\\]|$))' - , 'g'); -const reIsDeepProp = /\.|\[(?:[^[\]]*|(["'])(?:(?!\1)[^\\]|\\.)*?\1)\]/; -const reIsPlainProp = /^\w*$/; -const getTag = function(value){ - if(!value) - value === undefined ? '[object Undefined]' : '[object Null]'; - return Object.prototype.toString.call(value); -}; -const isSymbol = function(value){ - const type = typeof value; - return type === 'symbol' || (type === 'object' && value && getTag(value) === '[object Symbol]'); -}; -const isKey = function(value, object){ - if(Array.isArray(value)){ - return false; - } - const type = typeof value; - if(type === 'number' || type === 'symbol' || type === 'boolean' || !value || isSymbol(value)){ - return true; - } - return reIsPlainProp.test(value) || !reIsDeepProp.test(value) || - (object != null && value in Object(object)); -}; -const stringToPath = function(string){ - const result = []; - if(string.charCodeAt(0) === charCodeOfDot){ - result.push(''); - } - string.replace(rePropName, function(match, expression, quote, subString){ - let key = match; - if(quote){ - key = subString.replace(reEscapeChar, '$1'); - }else if(expression){ - key = expression.trim(); - } - result.push(key); - }); - return result; -}; -const castPath = function(value, object){ - if(Array.isArray(value)){ - return value; - } else { - return isKey(value, object) ? [value] : stringToPath(value); - } -}; -const toKey = function(value){ - if(typeof value === 'string' || isSymbol(value)) - return value; - const result = `${value}`; - // eslint-disable-next-line - return (result == '0' && (1 / value) == -INFINITY) ? '-0' : result; -}; -const get = function(object, path){ - path = castPath(path, object); - let index = 0; - const length = path.length; - while(object != null && index < length){ - object = object[toKey(path[index++])]; - } - return (index && index === length) ? object : undefined; -}; +import { CsvError } from './api/CsvError.js'; +import { is_object } from './utils/is_object.js'; +import { stringifier } from './api/index.js'; +import { normalize_options } from './api/normalize_options.js'; class Stringifier extends Transform { constructor(opts = {}){ super({...{writableObjectMode: true}, ...opts}); - const options = {}; - let err; - // Merge with user options - for(const opt in opts){ - options[underscore(opt)] = opts[opt]; - } - if((err = this.normalize(options)) !== undefined) throw err; - switch(options.record_delimiter){ - case 'auto': - options.record_delimiter = null; - break; - case 'unix': - options.record_delimiter = "\n"; - break; - case 'mac': - options.record_delimiter = "\r"; - break; - case 'windows': - options.record_delimiter = "\r\n"; - break; - case 'ascii': - options.record_delimiter = "\u001e"; - break; - case 'unicode': - options.record_delimiter = "\u2028"; - break; - } + const [err, options] = normalize_options(opts); + if(err !== undefined) throw err; // Expose options this.options = options; // Internal state @@ -155,159 +27,16 @@ class Stringifier extends Transform { this.info = { records: 0 }; - } - normalize(options){ - // Normalize option `bom` - if(options.bom === undefined || options.bom === null || options.bom === false){ - options.bom = false; - }else if(options.bom !== true){ - return new CsvError('CSV_OPTION_BOOLEAN_INVALID_TYPE', [ - 'option `bom` is optional and must be a boolean value,', - `got ${JSON.stringify(options.bom)}` - ]); - } - // Normalize option `delimiter` - if(options.delimiter === undefined || options.delimiter === null){ - options.delimiter = ','; - }else if(Buffer.isBuffer(options.delimiter)){ - options.delimiter = options.delimiter.toString(); - }else if(typeof options.delimiter !== 'string'){ - return new CsvError('CSV_OPTION_DELIMITER_INVALID_TYPE', [ - 'option `delimiter` must be a buffer or a string,', - `got ${JSON.stringify(options.delimiter)}` - ]); - } - // Normalize option `quote` - if(options.quote === undefined || options.quote === null){ - options.quote = '"'; - }else if(options.quote === true){ - options.quote = '"'; - }else if(options.quote === false){ - options.quote = ''; - }else if (Buffer.isBuffer(options.quote)){ - options.quote = options.quote.toString(); - }else if(typeof options.quote !== 'string'){ - return new CsvError('CSV_OPTION_QUOTE_INVALID_TYPE', [ - 'option `quote` must be a boolean, a buffer or a string,', - `got ${JSON.stringify(options.quote)}` - ]); - } - // Normalize option `quoted` - if(options.quoted === undefined || options.quoted === null){ - options.quoted = false; - }else{ - // todo - } - // Normalize option `quoted_empty` - if(options.quoted_empty === undefined || options.quoted_empty === null){ - options.quoted_empty = undefined; - }else{ - // todo - } - // Normalize option `quoted_match` - if(options.quoted_match === undefined || options.quoted_match === null || options.quoted_match === false){ - options.quoted_match = null; - }else if(!Array.isArray(options.quoted_match)){ - options.quoted_match = [options.quoted_match]; - } - if(options.quoted_match){ - for(const quoted_match of options.quoted_match){ - const isString = typeof quoted_match === 'string'; - const isRegExp = quoted_match instanceof RegExp; - if(!isString && !isRegExp){ - return Error(`Invalid Option: quoted_match must be a string or a regex, got ${JSON.stringify(quoted_match)}`); - } - } - } - // Normalize option `quoted_string` - if(options.quoted_string === undefined || options.quoted_string === null){ - options.quoted_string = false; - }else{ - // todo - } - // Normalize option `eof` - if(options.eof === undefined || options.eof === null){ - options.eof = true; - }else{ - // todo - } - // Normalize option `escape` - if(options.escape === undefined || options.escape === null){ - options.escape = '"'; - }else if(Buffer.isBuffer(options.escape)){ - options.escape = options.escape.toString(); - }else if(typeof options.escape !== 'string'){ - return Error(`Invalid Option: escape must be a buffer or a string, got ${JSON.stringify(options.escape)}`); - } - if (options.escape.length > 1){ - return Error(`Invalid Option: escape must be one character, got ${options.escape.length} characters`); - } - // Normalize option `header` - if(options.header === undefined || options.header === null){ - options.header = false; - }else{ - // todo - } - // Normalize option `columns` - const [err, columns] = this.normalize_columns(options.columns); - if(err) return err; - options.columns = columns; - // Normalize option `quoted` - if(options.quoted === undefined || options.quoted === null){ - options.quoted = false; - }else{ - // todo - } - // Normalize option `cast` - if(options.cast === undefined || options.cast === null){ - options.cast = {}; - }else{ - // todo - } - // Normalize option cast.bigint - if(options.cast.bigint === undefined || options.cast.bigint === null){ - // Cast boolean to string by default - options.cast.bigint = value => '' + value; - } - // Normalize option cast.boolean - if(options.cast.boolean === undefined || options.cast.boolean === null){ - // Cast boolean to string by default - options.cast.boolean = value => value ? '1' : ''; - } - // Normalize option cast.date - if(options.cast.date === undefined || options.cast.date === null){ - // Cast date to timestamp string by default - options.cast.date = value => '' + value.getTime(); - } - // Normalize option cast.number - if(options.cast.number === undefined || options.cast.number === null){ - // Cast number to string using native casting by default - options.cast.number = value => '' + value; - } - // Normalize option cast.object - if(options.cast.object === undefined || options.cast.object === null){ - // Stringify object as JSON by default - options.cast.object = value => JSON.stringify(value); - } - // Normalize option cast.string - if(options.cast.string === undefined || options.cast.string === null){ - // Leave string untouched - options.cast.string = function(value){return value;}; - } - // Normalize option `record_delimiter` - if(options.record_delimiter === undefined || options.record_delimiter === null){ - options.record_delimiter = '\n'; - }else if(Buffer.isBuffer(options.record_delimiter)){ - options.record_delimiter = options.record_delimiter.toString(); - }else if(typeof options.record_delimiter !== 'string'){ - return Error(`Invalid Option: record_delimiter must be a buffer or a string, got ${JSON.stringify(options.record_delimiter)}`); - } + this.api = stringifier(this.options, this.state, this.info); + this.api.options.on_record = (...args) => { + this.emit('record', ...args); + }; } _transform(chunk, encoding, callback){ if(this.state.stop === true){ return; } - const err = this.__transform(chunk); + const err = this.api.__transform(chunk, this.push.bind(this)); if(err !== undefined){ this.state.stop = true; } @@ -320,251 +49,12 @@ class Stringifier extends Transform { return; } if(this.info.records === 0){ - this.bom(); - const err = this.headers(); + this.api.bom(this.push.bind(this)); + const err = this.api.headers(this.push.bind(this)); if(err) callback(err); } callback(); } - __transform(chunk){ - // Chunk validation - if(!Array.isArray(chunk) && typeof chunk !== 'object'){ - return Error(`Invalid Record: expect an array or an object, got ${JSON.stringify(chunk)}`); - } - // Detect columns from the first record - if(this.info.records === 0){ - if(Array.isArray(chunk)){ - if(this.options.header === true && this.options.columns === undefined){ - return Error('Undiscoverable Columns: header option requires column option or object records'); - } - }else if(this.options.columns === undefined){ - const [err, columns] = this.normalize_columns(Object.keys(chunk)); - if(err) return; - this.options.columns = columns; - } - } - // Emit the header - if(this.info.records === 0){ - this.bom(); - const err = this.headers(); - if(err) return err; - } - // Emit and stringify the record if an object or an array - try{ - this.emit('record', chunk, this.info.records); - }catch(err){ - return err; - } - // Convert the record into a string - let err, chunk_string; - if(this.options.eof){ - [err, chunk_string] = this.stringify(chunk); - if(err) return err; - if(chunk_string === undefined){ - return; - }else{ - chunk_string = chunk_string + this.options.record_delimiter; - } - }else{ - [err, chunk_string] = this.stringify(chunk); - if(err) return err; - if(chunk_string === undefined){ - return; - }else{ - if(this.options.header || this.info.records){ - chunk_string = this.options.record_delimiter + chunk_string; - } - } - } - // Emit the csv - this.info.records++; - this.push(chunk_string); - } - stringify(chunk, chunkIsHeader=false){ - if(typeof chunk !== 'object'){ - return [undefined, chunk]; - } - const {columns} = this.options; - const record = []; - // Record is an array - if(Array.isArray(chunk)){ - // We are getting an array but the user has specified output columns. In - // this case, we respect the columns indexes - if(columns){ - chunk.splice(columns.length); - } - // Cast record elements - for(let i=0; i= 0; - const containsQuote = (quote !== '') && value.indexOf(quote) >= 0; - const containsEscape = value.indexOf(escape) >= 0 && (escape !== quote); - const containsRecordDelimiter = value.indexOf(record_delimiter) >= 0; - const quotedString = quoted_string && typeof field === 'string'; - let quotedMatch = quoted_match && quoted_match.filter(quoted_match => { - if(typeof quoted_match === 'string'){ - return value.indexOf(quoted_match) !== -1; - }else{ - return quoted_match.test(value); - } - }); - quotedMatch = quotedMatch && quotedMatch.length > 0; - const shouldQuote = containsQuote === true || containsdelimiter || containsRecordDelimiter || quoted || quotedString || quotedMatch; - if(shouldQuote === true && containsEscape === true){ - const regexp = escape === '\\' - ? new RegExp(escape + escape, 'g') - : new RegExp(escape, 'g'); - value = value.replace(regexp, escape + escape); - } - if(containsQuote === true){ - const regexp = new RegExp(quote,'g'); - value = value.replace(regexp, escape + quote); - } - if(shouldQuote === true){ - value = quote + value + quote; - } - csvrecord += value; - }else if(quoted_empty === true || (field === '' && quoted_string === true && quoted_empty !== false)){ - csvrecord += quote + quote; - } - if(i !== record.length - 1){ - csvrecord += delimiter; - } - } - return [undefined, csvrecord]; - } - bom(){ - if(this.options.bom !== true){ - return; - } - this.push(bom_utf8); - } - headers(){ - if(this.options.header === false){ - return; - } - if(this.options.columns === undefined){ - return; - } - let err; - let headers = this.options.columns.map(column => column.header); - if(this.options.eof){ - [err, headers] = this.stringify(headers, true); - headers += this.options.record_delimiter; - }else{ - [err, headers] = this.stringify(headers); - } - if(err) return err; - this.push(headers); - } - __cast(value, context){ - const type = typeof value; - try{ - if(type === 'string'){ // Fine for 99% of the cases - return [undefined, this.options.cast.string(value, context)]; - }else if(type === 'bigint'){ - return [undefined, this.options.cast.bigint(value, context)]; - }else if(type === 'number'){ - return [undefined, this.options.cast.number(value, context)]; - }else if(type === 'boolean'){ - return [undefined, this.options.cast.boolean(value, context)]; - }else if(value instanceof Date){ - return [undefined, this.options.cast.date(value, context)]; - }else if(type === 'object' && value !== null){ - return [undefined, this.options.cast.object(value, context)]; - }else{ - return [undefined, value, value]; - } - }catch(err){ - return [err]; - } - } - normalize_columns(columns){ - if(columns === undefined || columns === null){ - return []; - } - if(typeof columns !== 'object'){ - return [Error('Invalid option "columns": expect an array or an object')]; - } - if(!Array.isArray(columns)){ - const newcolumns = []; - for(const k in columns){ - newcolumns.push({ - key: k, - header: columns[k] - }); - } - columns = newcolumns; - }else{ - const newcolumns = []; - for(const column of columns){ - if(typeof column === 'string'){ - newcolumns.push({ - key: column, - header: column - }); - }else if(typeof column === 'object' && column !== undefined && !Array.isArray(column)){ - if(!column.key){ - return [Error('Invalid column definition: property "key" is required')]; - } - if(column.header === undefined){ - column.header = column.key; - } - newcolumns.push(column); - }else{ - return [Error('Invalid column definition: expect a string or an object')]; - } - } - columns = newcolumns; - } - return [undefined, columns]; - } } const stringify = function(){ @@ -574,7 +64,7 @@ const stringify = function(){ const type = typeof argument; if(data === undefined && (Array.isArray(argument))){ data = argument; - }else if(options === undefined && isObject(argument)){ + }else if(options === undefined && is_object(argument)){ options = argument; }else if(callback === undefined && type === 'function'){ callback = argument; diff --git a/packages/csv-stringify/lib/sync.js b/packages/csv-stringify/lib/sync.js index 2251947d5..c140a3be2 100644 --- a/packages/csv-stringify/lib/sync.js +++ b/packages/csv-stringify/lib/sync.js @@ -1,17 +1,29 @@ -import { Stringifier } from './index.js'; +import { stringifier } from './api/index.js'; +import { normalize_options } from './api/normalize_options.js'; -const stringify = function(records, options={}){ +const stringify = function(records, opts={}){ const data = []; - const stringifier = new Stringifier(options); - stringifier.push = function(record){ - if(record === null){ - return; - } - data.push(record.toString()); + const [err, options] = normalize_options(opts); + if(err !== undefined) throw err; + const state = { + stop: false }; + // Information + const info = { + records: 0 + }; + const api = stringifier(options, state, info); + // stringifier.push = function(record){ + // if(record === null){ + // return; + // } + // data.push(record.toString()); + // }; for(const record of records){ - const err = stringifier.__transform(record, null); + const err = api.__transform(record, function(record){ + data.push(record); + }); if(err !== undefined) throw err; } return data.join(''); diff --git a/packages/csv-stringify/lib/utils/get.js b/packages/csv-stringify/lib/utils/get.js new file mode 100644 index 000000000..5dd9effaa --- /dev/null +++ b/packages/csv-stringify/lib/utils/get.js @@ -0,0 +1,82 @@ + + +// Lodash implementation of `get` + +const charCodeOfDot = '.'.charCodeAt(0); +const reEscapeChar = /\\(\\)?/g; +const rePropName = RegExp( + // Match anything that isn't a dot or bracket. + '[^.[\\]]+' + '|' + + // Or match property names within brackets. + '\\[(?:' + + // Match a non-string expression. + '([^"\'][^[]*)' + '|' + + // Or match strings (supports escaping characters). + '(["\'])((?:(?!\\2)[^\\\\]|\\\\.)*?)\\2' + + ')\\]'+ '|' + + // Or match "" as the space between consecutive dots or empty brackets. + '(?=(?:\\.|\\[\\])(?:\\.|\\[\\]|$))' + , 'g'); +const reIsDeepProp = /\.|\[(?:[^[\]]*|(["'])(?:(?!\1)[^\\]|\\.)*?\1)\]/; +const reIsPlainProp = /^\w*$/; +const getTag = function(value){ + if(!value) + value === undefined ? '[object Undefined]' : '[object Null]'; + return Object.prototype.toString.call(value); +}; +const isSymbol = function(value){ + const type = typeof value; + return type === 'symbol' || (type === 'object' && value && getTag(value) === '[object Symbol]'); +}; +const isKey = function(value, object){ + if(Array.isArray(value)){ + return false; + } + const type = typeof value; + if(type === 'number' || type === 'symbol' || type === 'boolean' || !value || isSymbol(value)){ + return true; + } + return reIsPlainProp.test(value) || !reIsDeepProp.test(value) || + (object != null && value in Object(object)); +}; +const stringToPath = function(string){ + const result = []; + if(string.charCodeAt(0) === charCodeOfDot){ + result.push(''); + } + string.replace(rePropName, function(match, expression, quote, subString){ + let key = match; + if(quote){ + key = subString.replace(reEscapeChar, '$1'); + }else if(expression){ + key = expression.trim(); + } + result.push(key); + }); + return result; +}; +const castPath = function(value, object){ + if(Array.isArray(value)){ + return value; + } else { + return isKey(value, object) ? [value] : stringToPath(value); + } +}; +const toKey = function(value){ + if(typeof value === 'string' || isSymbol(value)) + return value; + const result = `${value}`; + // eslint-disable-next-line + return (result == '0' && (1 / value) == -INFINITY) ? '-0' : result; +}; +const get = function(object, path){ + path = castPath(path, object); + let index = 0; + const length = path.length; + while(object != null && index < length){ + object = object[toKey(path[index++])]; + } + return (index && index === length) ? object : undefined; +}; + +export {get}; diff --git a/packages/csv-stringify/lib/utils/is_object.js b/packages/csv-stringify/lib/utils/is_object.js new file mode 100644 index 000000000..13c243bc6 --- /dev/null +++ b/packages/csv-stringify/lib/utils/is_object.js @@ -0,0 +1,7 @@ + + +const is_object = function(obj){ + return typeof obj === 'object' && obj !== null && ! Array.isArray(obj); +}; + +export {is_object}; diff --git a/packages/csv-stringify/lib/utils/underscore.js b/packages/csv-stringify/lib/utils/underscore.js new file mode 100644 index 000000000..78397a32e --- /dev/null +++ b/packages/csv-stringify/lib/utils/underscore.js @@ -0,0 +1,9 @@ + + +const underscore = function(str){ + return str.replace(/([A-Z])/g, function(_, match){ + return '_' + match.toLowerCase(); + }); +}; + +export { underscore }; diff --git a/packages/csv-stringify/test/api.types.ts b/packages/csv-stringify/test/api.types.ts index 59a86acde..10f047606 100644 --- a/packages/csv-stringify/test/api.types.ts +++ b/packages/csv-stringify/test/api.types.ts @@ -12,7 +12,7 @@ describe('API Types', () => { const keys: any = Object.keys(options) keys.sort().should.eql([ 'bom', 'cast', 'columns', 'delimiter', 'eof', 'escape', - 'header', 'quote', 'quoted', 'quoted_empty', + 'header', 'on_record', 'quote', 'quoted', 'quoted_empty', 'quoted_match', 'quoted_string', 'record_delimiter' ]) }) diff --git a/packages/csv-stringify/test/api.web_stream.coffee b/packages/csv-stringify/test/api.web_stream.coffee new file mode 100644 index 000000000..5434ad5f3 --- /dev/null +++ b/packages/csv-stringify/test/api.web_stream.coffee @@ -0,0 +1,18 @@ + +# import {generate as generateStream} from 'csv-generate/stream' +# import {stringify as stringifyStream} from '../lib/stream.js' +# import {stringify as stringifyClassic} from '../lib/index.js' +# +# describe 'api stream', -> +# +# it.skip 'perf stream with iterator', -> +# generator = generateStream +# objectMode: true, +# length: 5 +# stringifier = stringifyStream() +# stream = generator.pipeThrough stringifier +# chunks = [] +# for await chunk from stream +# chunks.push chunk +# # records.length.should.eql 5 +# console.log(chunks) diff --git a/packages/csv/dist/cjs/index.cjs b/packages/csv/dist/cjs/index.cjs index 4dba281f5..3f9fbc8a6 100644 --- a/packages/csv/dist/cjs/index.cjs +++ b/packages/csv/dist/cjs/index.cjs @@ -10,20 +10,64 @@ function _interopDefaultLegacy (e) { return e && typeof e === 'object' && 'defau var stream__default = /*#__PURE__*/_interopDefaultLegacy(stream); var util__default = /*#__PURE__*/_interopDefaultLegacy(util); -const Generator = function(options = {}){ +const init_state$1 = (options) => { + // State + return { + start_time: options.duration ? Date.now() : null, + fixed_size_buffer: '', + count_written: 0, + count_created: 0, + }; +}; + +// Generate a random number between 0 and 1 with 2 decimals. The function is idempotent if it detect the "seed" option. +const random = function(options={}){ + if(options.seed){ + return options.seed = options.seed * Math.PI * 100 % 100 / 100; + }else { + return Math.random(); + } +}; + +const types = { + // Generate an ASCII value. + ascii: function({options}){ + const column = []; + const nb_chars = Math.ceil(random(options) * options.maxWordLength); + for(let i=0; i { // Convert Stream Readable options if underscored - if(options.high_water_mark){ - options.highWaterMark = options.high_water_mark; + if(opts.high_water_mark){ + opts.highWaterMark = opts.high_water_mark; } - if(options.object_mode){ - options.objectMode = options.object_mode; + if(opts.object_mode){ + opts.objectMode = opts.object_mode; } - // Call parent constructor - stream__default["default"].Readable.call(this, options); // Clone and camelize options - this.options = {}; - for(const k in options){ - this.options[Generator.camelize(k)] = options[k]; + const options = {}; + for(const k in opts){ + options[camelize(k)] = opts[k]; } // Normalize options const dft = { @@ -41,110 +85,100 @@ const Generator = function(options = {}){ sleep: 0, }; for(const k in dft){ - if(this.options[k] === undefined){ - this.options[k] = dft[k]; + if(options[k] === undefined){ + options[k] = dft[k]; } } // Default values - if(this.options.eof === true){ - this.options.eof = this.options.rowDelimiter; + if(options.eof === true){ + options.eof = options.rowDelimiter; } - // State - this._ = { - start_time: this.options.duration ? Date.now() : null, - fixed_size_buffer: '', - count_written: 0, - count_created: 0, - }; - if(typeof this.options.columns === 'number'){ - this.options.columns = new Array(this.options.columns); + if(typeof options.columns === 'number'){ + options.columns = new Array(options.columns); } - const accepted_header_types = Object.keys(Generator).filter((t) => (!['super_', 'camelize'].includes(t))); - for(let i = 0; i < this.options.columns.length; i++){ - const v = this.options.columns[i] || 'ascii'; + const accepted_header_types = Object.keys(types).filter((t) => (!['super_', 'camelize'].includes(t))); + for(let i = 0; i < options.columns.length; i++){ + const v = options.columns[i] || 'ascii'; if(typeof v === 'string'){ if(!accepted_header_types.includes(v)){ throw Error(`Invalid column type: got "${v}", default values are ${JSON.stringify(accepted_header_types)}`); } - this.options.columns[i] = Generator[v]; + options.columns[i] = types[v]; } } - return this; + return options; }; -util__default["default"].inherits(Generator, stream__default["default"].Readable); -// Generate a random number between 0 and 1 with 2 decimals. The function is idempotent if it detect the "seed" option. -Generator.prototype.random = function(){ - if(this.options.seed){ - return this.options.seed = this.options.seed * Math.PI * 100 % 100 / 100; - }else { - return Math.random(); - } -}; -// Stop the generation. -Generator.prototype.end = function(){ - this.push(null); -}; -// Put new data into the read queue. -Generator.prototype._read = function(size){ +const read = (options, state, size, push, close) => { // Already started const data = []; - let length = this._.fixed_size_buffer.length; + let length = state.fixed_size_buffer.length; if(length !== 0){ - data.push(this._.fixed_size_buffer); + data.push(state.fixed_size_buffer); } // eslint-disable-next-line while(true){ // Time for some rest: flush first and stop later - if((this._.count_created === this.options.length) || (this.options.end && Date.now() > this.options.end) || (this.options.duration && Date.now() > this._.start_time + this.options.duration)){ + if((state.count_created === options.length) || (options.end && Date.now() > options.end) || (options.duration && Date.now() > state.start_time + options.duration)){ // Flush if(data.length){ - if(this.options.objectMode){ + if(options.objectMode){ for(const record of data){ - this.__push(record); + push(record); } }else { - this.__push(data.join('') + (this.options.eof ? this.options.eof : '')); + push(data.join('') + (options.eof ? options.eof : '')); } - this._.end = true; + state.end = true; }else { - this.push(null); + close(); } return; } // Create the record let record = []; let recordLength; - this.options.columns.forEach((fn) => { - record.push(fn(this)); + options.columns.forEach((fn) => { + const result = fn({options: options, state: state}); + const type = typeof result; + if(result !== null && type !== 'string' && type !== 'number'){ + throw Error([ + 'INVALID_VALUE:', + 'values returned by column function must be', + 'a string, a number or null,', + `got ${JSON.stringify(result)}` + ].join(' ')); + } + record.push(result); }); // Obtain record length - if(this.options.objectMode){ + if(options.objectMode){ recordLength = 0; // recordLength is currently equal to the number of columns // This is wrong and shall equal to 1 record only - for(const column of record) + for(const column of record){ recordLength += column.length; + } }else { // Stringify the record - record = (this._.count_created === 0 ? '' : this.options.rowDelimiter)+record.join(this.options.delimiter); + record = (state.count_created === 0 ? '' : options.rowDelimiter)+record.join(options.delimiter); recordLength = record.length; } - this._.count_created++; + state.count_created++; if(length + recordLength > size){ - if(this.options.objectMode){ + if(options.objectMode){ data.push(record); for(const record of data){ - this.__push(record); + push(record); } }else { - if(this.options.fixedSize){ - this._.fixed_size_buffer = record.substr(size - length); + if(options.fixedSize){ + state.fixed_size_buffer = record.substr(size - length); data.push(record.substr(0, size - length)); }else { data.push(record); } - this.__push(data.join('')); + push(data.join('')); } return; } @@ -152,42 +186,41 @@ Generator.prototype._read = function(size){ data.push(record); } }; + +const Generator = function(options = {}){ + this.options = normalize_options$2(options); + // Call parent constructor + stream__default["default"].Readable.call(this, this.options); + this.state = init_state$1(this.options); + return this; +}; +util__default["default"].inherits(Generator, stream__default["default"].Readable); + +// Stop the generation. +Generator.prototype.end = function(){ + this.push(null); +}; +// Put new data into the read queue. +Generator.prototype._read = function(size){ + const self = this; + read(this.options, this.state, size, function(chunk) { + self.__push(chunk); + }, function(){ + self.push(null); + }); +}; // Put new data into the read queue. Generator.prototype.__push = function(record){ + // console.log('push', record) const push = () => { - this._.count_written++; + this.state.count_written++; this.push(record); - if(this._.end === true){ + if(this.state.end === true){ return this.push(null); } }; this.options.sleep > 0 ? setTimeout(push, this.options.sleep) : push(); }; -// Generate an ASCII value. -Generator.ascii = function(gen){ - // Column - const column = []; - const nb_chars = Math.ceil(gen.random() * gen.options.maxWordLength); - for(let i=0; i delimiter.length), + // Skip if the remaining buffer can be escape sequence + options.quote !== null ? options.quote.length : 0, + ), + previousBuf: undefined, + quoting: false, + stop: false, + rawBuffer: new ResizeableBuffer(100), + record: [], + recordHasError: false, + record_length: 0, + recordDelimiterMaxLength: options.record_delimiter.length === 0 ? 2 : Math.max(...options.record_delimiter.map((v) => v.length)), + trimChars: [Buffer.from(' ', options.encoding)[0], Buffer.from('\t', options.encoding)[0]], + wasQuoting: false, + wasRowDelimiter: false + }; }; -class CsvError$1 extends Error { - constructor(code, message, options, ...contexts) { - if(Array.isArray(message)) message = message.join(' '); - super(message); - if(Error.captureStackTrace !== undefined){ - Error.captureStackTrace(this, CsvError$1); - } - this.code = code; - for(const context of contexts){ - for(const key in context){ - const value = context[key]; - this[key] = Buffer.isBuffer(value) ? value.toString(options.encoding) : value == null ? value : JSON.parse(JSON.stringify(value)); - } - } - } -} - const underscore$1 = function(str){ return str.replace(/([A-Z])/g, function(_, match){ return '_' + match.toLowerCase(); }); }; -const isObject$1 = function(obj){ - return (typeof obj === 'object' && obj !== null && !Array.isArray(obj)); -}; - -const isRecordEmpty = function(record){ - return record.every((field) => field == null || field.toString && field.toString().trim() === ''); -}; - -const normalizeColumnsArray = function(columns){ - const normalizedColumns = []; - for(let i = 0, l = columns.length; i < l; i++){ - const column = columns[i]; - if(column === undefined || column === null || column === false){ - normalizedColumns[i] = { disabled: true }; - }else if(typeof column === 'string'){ - normalizedColumns[i] = { name: column }; - }else if(isObject$1(column)){ - if(typeof column.name !== 'string'){ - throw new CsvError$1('CSV_OPTION_COLUMNS_MISSING_NAME', [ - 'Option columns missing name:', - `property "name" is required at position ${i}`, - 'when column is an object literal' - ]); - } - normalizedColumns[i] = column; - }else { - throw new CsvError$1('CSV_INVALID_COLUMN_DEFINITION', [ - 'Invalid column definition:', - 'expect a string or a literal object,', - `got ${JSON.stringify(column)} at position ${i}` - ]); - } - } - return normalizedColumns; -}; - -class Parser extends stream.Transform { - constructor(opts = {}){ - super({...{readableObjectMode: true}, ...opts, encoding: null}); - this.__originalOptions = opts; - this.__normalizeOptions(opts); - } - __normalizeOptions(opts){ - const options = {}; - // Merge with user options - for(const opt in opts){ - options[underscore$1(opt)] = opts[opt]; - } - // Normalize option `encoding` - // Note: defined first because other options depends on it - // to convert chars/strings into buffers. - if(options.encoding === undefined || options.encoding === true){ - options.encoding = 'utf8'; - }else if(options.encoding === null || options.encoding === false){ - options.encoding = null; - }else if(typeof options.encoding !== 'string' && options.encoding !== null){ - throw new CsvError$1('CSV_INVALID_OPTION_ENCODING', [ - 'Invalid option encoding:', - 'encoding must be a string or null to return a buffer,', - `got ${JSON.stringify(options.encoding)}` - ], options); - } - // Normalize option `bom` - if(options.bom === undefined || options.bom === null || options.bom === false){ - options.bom = false; - }else if(options.bom !== true){ - throw new CsvError$1('CSV_INVALID_OPTION_BOM', [ - 'Invalid option bom:', 'bom must be true,', - `got ${JSON.stringify(options.bom)}` - ], options); - } - // Normalize option `cast` - let fnCastField = null; - if(options.cast === undefined || options.cast === null || options.cast === false || options.cast === ''){ - options.cast = undefined; - }else if(typeof options.cast === 'function'){ - fnCastField = options.cast; - options.cast = true; - }else if(options.cast !== true){ - throw new CsvError$1('CSV_INVALID_OPTION_CAST', [ - 'Invalid option cast:', 'cast must be true or a function,', - `got ${JSON.stringify(options.cast)}` - ], options); - } - // Normalize option `cast_date` - if(options.cast_date === undefined || options.cast_date === null || options.cast_date === false || options.cast_date === ''){ - options.cast_date = false; - }else if(options.cast_date === true){ - options.cast_date = function(value){ - const date = Date.parse(value); - return !isNaN(date) ? new Date(date) : value; - }; - }else { - throw new CsvError$1('CSV_INVALID_OPTION_CAST_DATE', [ - 'Invalid option cast_date:', 'cast_date must be true or a function,', - `got ${JSON.stringify(options.cast_date)}` - ], options); - } - // Normalize option `columns` - let fnFirstLineToHeaders = null; - if(options.columns === true){ - // Fields in the first line are converted as-is to columns - fnFirstLineToHeaders = undefined; - }else if(typeof options.columns === 'function'){ - fnFirstLineToHeaders = options.columns; - options.columns = true; - }else if(Array.isArray(options.columns)){ - options.columns = normalizeColumnsArray(options.columns); - }else if(options.columns === undefined || options.columns === null || options.columns === false){ - options.columns = false; - }else { - throw new CsvError$1('CSV_INVALID_OPTION_COLUMNS', [ - 'Invalid option columns:', - 'expect an array, a function or true,', - `got ${JSON.stringify(options.columns)}` +const normalize_options$1 = function(opts){ + const options = {}; + // Merge with user options + for(const opt in opts){ + options[underscore$1(opt)] = opts[opt]; + } + // Normalize option `encoding` + // Note: defined first because other options depends on it + // to convert chars/strings into buffers. + if(options.encoding === undefined || options.encoding === true){ + options.encoding = 'utf8'; + }else if(options.encoding === null || options.encoding === false){ + options.encoding = null; + }else if(typeof options.encoding !== 'string' && options.encoding !== null){ + throw new CsvError$1('CSV_INVALID_OPTION_ENCODING', [ + 'Invalid option encoding:', + 'encoding must be a string or null to return a buffer,', + `got ${JSON.stringify(options.encoding)}` + ], options); + } + // Normalize option `bom` + if(options.bom === undefined || options.bom === null || options.bom === false){ + options.bom = false; + }else if(options.bom !== true){ + throw new CsvError$1('CSV_INVALID_OPTION_BOM', [ + 'Invalid option bom:', 'bom must be true,', + `got ${JSON.stringify(options.bom)}` + ], options); + } + // Normalize option `cast` + options.cast_function = null; + if(options.cast === undefined || options.cast === null || options.cast === false || options.cast === ''){ + options.cast = undefined; + }else if(typeof options.cast === 'function'){ + options.cast_function = options.cast; + options.cast = true; + }else if(options.cast !== true){ + throw new CsvError$1('CSV_INVALID_OPTION_CAST', [ + 'Invalid option cast:', 'cast must be true or a function,', + `got ${JSON.stringify(options.cast)}` + ], options); + } + // Normalize option `cast_date` + if(options.cast_date === undefined || options.cast_date === null || options.cast_date === false || options.cast_date === ''){ + options.cast_date = false; + }else if(options.cast_date === true){ + options.cast_date = function(value){ + const date = Date.parse(value); + return !isNaN(date) ? new Date(date) : value; + }; + }else { + throw new CsvError$1('CSV_INVALID_OPTION_CAST_DATE', [ + 'Invalid option cast_date:', 'cast_date must be true or a function,', + `got ${JSON.stringify(options.cast_date)}` + ], options); + } + // Normalize option `columns` + options.cast_first_line_to_header = null; + if(options.columns === true){ + // Fields in the first line are converted as-is to columns + options.cast_first_line_to_header = undefined; + }else if(typeof options.columns === 'function'){ + options.cast_first_line_to_header = options.columns; + options.columns = true; + }else if(Array.isArray(options.columns)){ + options.columns = normalize_columns_array(options.columns); + }else if(options.columns === undefined || options.columns === null || options.columns === false){ + options.columns = false; + }else { + throw new CsvError$1('CSV_INVALID_OPTION_COLUMNS', [ + 'Invalid option columns:', + 'expect an array, a function or true,', + `got ${JSON.stringify(options.columns)}` + ], options); + } + // Normalize option `group_columns_by_name` + if(options.group_columns_by_name === undefined || options.group_columns_by_name === null || options.group_columns_by_name === false){ + options.group_columns_by_name = false; + }else if(options.group_columns_by_name !== true){ + throw new CsvError$1('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ + 'Invalid option group_columns_by_name:', + 'expect an boolean,', + `got ${JSON.stringify(options.group_columns_by_name)}` + ], options); + }else if(options.columns === false){ + throw new CsvError$1('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ + 'Invalid option group_columns_by_name:', + 'the `columns` mode must be activated.' + ], options); + } + // Normalize option `comment` + if(options.comment === undefined || options.comment === null || options.comment === false || options.comment === ''){ + options.comment = null; + }else { + if(typeof options.comment === 'string'){ + options.comment = Buffer.from(options.comment, options.encoding); + } + if(!Buffer.isBuffer(options.comment)){ + throw new CsvError$1('CSV_INVALID_OPTION_COMMENT', [ + 'Invalid option comment:', + 'comment must be a buffer or a string,', + `got ${JSON.stringify(options.comment)}` ], options); } - // Normalize option `group_columns_by_name` - if(options.group_columns_by_name === undefined || options.group_columns_by_name === null || options.group_columns_by_name === false){ - options.group_columns_by_name = false; - }else if(options.group_columns_by_name !== true){ - throw new CsvError$1('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ - 'Invalid option group_columns_by_name:', - 'expect an boolean,', - `got ${JSON.stringify(options.group_columns_by_name)}` - ], options); - }else if(options.columns === false){ - throw new CsvError$1('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ - 'Invalid option group_columns_by_name:', - 'the `columns` mode must be activated.' - ], options); + } + // Normalize option `delimiter` + const delimiter_json = JSON.stringify(options.delimiter); + if(!Array.isArray(options.delimiter)) options.delimiter = [options.delimiter]; + if(options.delimiter.length === 0){ + throw new CsvError$1('CSV_INVALID_OPTION_DELIMITER', [ + 'Invalid option delimiter:', + 'delimiter must be a non empty string or buffer or array of string|buffer,', + `got ${delimiter_json}` + ], options); + } + options.delimiter = options.delimiter.map(function(delimiter){ + if(delimiter === undefined || delimiter === null || delimiter === false){ + return Buffer.from(',', options.encoding); } - // Normalize option `comment` - if(options.comment === undefined || options.comment === null || options.comment === false || options.comment === ''){ - options.comment = null; - }else { - if(typeof options.comment === 'string'){ - options.comment = Buffer.from(options.comment, options.encoding); - } - if(!Buffer.isBuffer(options.comment)){ - throw new CsvError$1('CSV_INVALID_OPTION_COMMENT', [ - 'Invalid option comment:', - 'comment must be a buffer or a string,', - `got ${JSON.stringify(options.comment)}` - ], options); - } + if(typeof delimiter === 'string'){ + delimiter = Buffer.from(delimiter, options.encoding); } - // Normalize option `delimiter` - const delimiter_json = JSON.stringify(options.delimiter); - if(!Array.isArray(options.delimiter)) options.delimiter = [options.delimiter]; - if(options.delimiter.length === 0){ + if(!Buffer.isBuffer(delimiter) || delimiter.length === 0){ throw new CsvError$1('CSV_INVALID_OPTION_DELIMITER', [ 'Invalid option delimiter:', 'delimiter must be a non empty string or buffer or array of string|buffer,', `got ${delimiter_json}` ], options); } - options.delimiter = options.delimiter.map(function(delimiter){ - if(delimiter === undefined || delimiter === null || delimiter === false){ - return Buffer.from(',', options.encoding); - } - if(typeof delimiter === 'string'){ - delimiter = Buffer.from(delimiter, options.encoding); - } - if(!Buffer.isBuffer(delimiter) || delimiter.length === 0){ - throw new CsvError$1('CSV_INVALID_OPTION_DELIMITER', [ - 'Invalid option delimiter:', - 'delimiter must be a non empty string or buffer or array of string|buffer,', - `got ${delimiter_json}` - ], options); + return delimiter; + }); + // Normalize option `escape` + if(options.escape === undefined || options.escape === true){ + options.escape = Buffer.from('"', options.encoding); + }else if(typeof options.escape === 'string'){ + options.escape = Buffer.from(options.escape, options.encoding); + }else if (options.escape === null || options.escape === false){ + options.escape = null; + } + if(options.escape !== null){ + if(!Buffer.isBuffer(options.escape)){ + throw new Error(`Invalid Option: escape must be a buffer, a string or a boolean, got ${JSON.stringify(options.escape)}`); + } + } + // Normalize option `from` + if(options.from === undefined || options.from === null){ + options.from = 1; + }else { + if(typeof options.from === 'string' && /\d+/.test(options.from)){ + options.from = parseInt(options.from); + } + if(Number.isInteger(options.from)){ + if(options.from < 0){ + throw new Error(`Invalid Option: from must be a positive integer, got ${JSON.stringify(opts.from)}`); } - return delimiter; - }); - // Normalize option `escape` - if(options.escape === undefined || options.escape === true){ - options.escape = Buffer.from('"', options.encoding); - }else if(typeof options.escape === 'string'){ - options.escape = Buffer.from(options.escape, options.encoding); - }else if (options.escape === null || options.escape === false){ - options.escape = null; - } - if(options.escape !== null){ - if(!Buffer.isBuffer(options.escape)){ - throw new Error(`Invalid Option: escape must be a buffer, a string or a boolean, got ${JSON.stringify(options.escape)}`); - } - } - // Normalize option `from` - if(options.from === undefined || options.from === null){ - options.from = 1; }else { - if(typeof options.from === 'string' && /\d+/.test(options.from)){ - options.from = parseInt(options.from); - } - if(Number.isInteger(options.from)){ - if(options.from < 0){ - throw new Error(`Invalid Option: from must be a positive integer, got ${JSON.stringify(opts.from)}`); - } - }else { - throw new Error(`Invalid Option: from must be an integer, got ${JSON.stringify(options.from)}`); - } + throw new Error(`Invalid Option: from must be an integer, got ${JSON.stringify(options.from)}`); } - // Normalize option `from_line` - if(options.from_line === undefined || options.from_line === null){ - options.from_line = 1; - }else { - if(typeof options.from_line === 'string' && /\d+/.test(options.from_line)){ - options.from_line = parseInt(options.from_line); - } - if(Number.isInteger(options.from_line)){ - if(options.from_line <= 0){ - throw new Error(`Invalid Option: from_line must be a positive integer greater than 0, got ${JSON.stringify(opts.from_line)}`); - } - }else { - throw new Error(`Invalid Option: from_line must be an integer, got ${JSON.stringify(opts.from_line)}`); + } + // Normalize option `from_line` + if(options.from_line === undefined || options.from_line === null){ + options.from_line = 1; + }else { + if(typeof options.from_line === 'string' && /\d+/.test(options.from_line)){ + options.from_line = parseInt(options.from_line); + } + if(Number.isInteger(options.from_line)){ + if(options.from_line <= 0){ + throw new Error(`Invalid Option: from_line must be a positive integer greater than 0, got ${JSON.stringify(opts.from_line)}`); } + }else { + throw new Error(`Invalid Option: from_line must be an integer, got ${JSON.stringify(opts.from_line)}`); } - // Normalize options `ignore_last_delimiters` - if(options.ignore_last_delimiters === undefined || options.ignore_last_delimiters === null){ + } + // Normalize options `ignore_last_delimiters` + if(options.ignore_last_delimiters === undefined || options.ignore_last_delimiters === null){ + options.ignore_last_delimiters = false; + }else if(typeof options.ignore_last_delimiters === 'number'){ + options.ignore_last_delimiters = Math.floor(options.ignore_last_delimiters); + if(options.ignore_last_delimiters === 0){ options.ignore_last_delimiters = false; - }else if(typeof options.ignore_last_delimiters === 'number'){ - options.ignore_last_delimiters = Math.floor(options.ignore_last_delimiters); - if(options.ignore_last_delimiters === 0){ - options.ignore_last_delimiters = false; - } - }else if(typeof options.ignore_last_delimiters !== 'boolean'){ - throw new CsvError$1('CSV_INVALID_OPTION_IGNORE_LAST_DELIMITERS', [ - 'Invalid option `ignore_last_delimiters`:', - 'the value must be a boolean value or an integer,', - `got ${JSON.stringify(options.ignore_last_delimiters)}` - ], options); } - if(options.ignore_last_delimiters === true && options.columns === false){ - throw new CsvError$1('CSV_IGNORE_LAST_DELIMITERS_REQUIRES_COLUMNS', [ - 'The option `ignore_last_delimiters`', - 'requires the activation of the `columns` option' + }else if(typeof options.ignore_last_delimiters !== 'boolean'){ + throw new CsvError$1('CSV_INVALID_OPTION_IGNORE_LAST_DELIMITERS', [ + 'Invalid option `ignore_last_delimiters`:', + 'the value must be a boolean value or an integer,', + `got ${JSON.stringify(options.ignore_last_delimiters)}` + ], options); + } + if(options.ignore_last_delimiters === true && options.columns === false){ + throw new CsvError$1('CSV_IGNORE_LAST_DELIMITERS_REQUIRES_COLUMNS', [ + 'The option `ignore_last_delimiters`', + 'requires the activation of the `columns` option' + ], options); + } + // Normalize option `info` + if(options.info === undefined || options.info === null || options.info === false){ + options.info = false; + }else if(options.info !== true){ + throw new Error(`Invalid Option: info must be true, got ${JSON.stringify(options.info)}`); + } + // Normalize option `max_record_size` + if(options.max_record_size === undefined || options.max_record_size === null || options.max_record_size === false){ + options.max_record_size = 0; + }else if(Number.isInteger(options.max_record_size) && options.max_record_size >= 0);else if(typeof options.max_record_size === 'string' && /\d+/.test(options.max_record_size)){ + options.max_record_size = parseInt(options.max_record_size); + }else { + throw new Error(`Invalid Option: max_record_size must be a positive integer, got ${JSON.stringify(options.max_record_size)}`); + } + // Normalize option `objname` + if(options.objname === undefined || options.objname === null || options.objname === false){ + options.objname = undefined; + }else if(Buffer.isBuffer(options.objname)){ + if(options.objname.length === 0){ + throw new Error(`Invalid Option: objname must be a non empty buffer`); + } + if(options.encoding === null);else { + options.objname = options.objname.toString(options.encoding); + } + }else if(typeof options.objname === 'string'){ + if(options.objname.length === 0){ + throw new Error(`Invalid Option: objname must be a non empty string`); + } + // Great, nothing to do + }else if(typeof options.objname === 'number');else { + throw new Error(`Invalid Option: objname must be a string or a buffer, got ${options.objname}`); + } + if(options.objname !== undefined){ + if(typeof options.objname === 'number'){ + if(options.columns !== false){ + throw Error('Invalid Option: objname index cannot be combined with columns or be defined as a field'); + } + }else { // A string or a buffer + if(options.columns === false){ + throw Error('Invalid Option: objname field must be combined with columns or be defined as an index'); + } + } + } + // Normalize option `on_record` + if(options.on_record === undefined || options.on_record === null){ + options.on_record = undefined; + }else if(typeof options.on_record !== 'function'){ + throw new CsvError$1('CSV_INVALID_OPTION_ON_RECORD', [ + 'Invalid option `on_record`:', + 'expect a function,', + `got ${JSON.stringify(options.on_record)}` + ], options); + } + // Normalize option `quote` + if(options.quote === null || options.quote === false || options.quote === ''){ + options.quote = null; + }else { + if(options.quote === undefined || options.quote === true){ + options.quote = Buffer.from('"', options.encoding); + }else if(typeof options.quote === 'string'){ + options.quote = Buffer.from(options.quote, options.encoding); + } + if(!Buffer.isBuffer(options.quote)){ + throw new Error(`Invalid Option: quote must be a buffer or a string, got ${JSON.stringify(options.quote)}`); + } + } + // Normalize option `raw` + if(options.raw === undefined || options.raw === null || options.raw === false){ + options.raw = false; + }else if(options.raw !== true){ + throw new Error(`Invalid Option: raw must be true, got ${JSON.stringify(options.raw)}`); + } + // Normalize option `record_delimiter` + if(options.record_delimiter === undefined){ + options.record_delimiter = []; + }else if(typeof options.record_delimiter === 'string' || Buffer.isBuffer(options.record_delimiter)){ + if(options.record_delimiter.length === 0){ + throw new CsvError$1('CSV_INVALID_OPTION_RECORD_DELIMITER', [ + 'Invalid option `record_delimiter`:', + 'value must be a non empty string or buffer,', + `got ${JSON.stringify(options.record_delimiter)}` ], options); } - // Normalize option `info` - if(options.info === undefined || options.info === null || options.info === false){ - options.info = false; - }else if(options.info !== true){ - throw new Error(`Invalid Option: info must be true, got ${JSON.stringify(options.info)}`); - } - // Normalize option `max_record_size` - if(options.max_record_size === undefined || options.max_record_size === null || options.max_record_size === false){ - options.max_record_size = 0; - }else if(Number.isInteger(options.max_record_size) && options.max_record_size >= 0);else if(typeof options.max_record_size === 'string' && /\d+/.test(options.max_record_size)){ - options.max_record_size = parseInt(options.max_record_size); - }else { - throw new Error(`Invalid Option: max_record_size must be a positive integer, got ${JSON.stringify(options.max_record_size)}`); - } - // Normalize option `objname` - if(options.objname === undefined || options.objname === null || options.objname === false){ - options.objname = undefined; - }else if(Buffer.isBuffer(options.objname)){ - if(options.objname.length === 0){ - throw new Error(`Invalid Option: objname must be a non empty buffer`); - } - if(options.encoding === null);else { - options.objname = options.objname.toString(options.encoding); - } - }else if(typeof options.objname === 'string'){ - if(options.objname.length === 0){ - throw new Error(`Invalid Option: objname must be a non empty string`); - } - // Great, nothing to do - }else if(typeof options.objname === 'number');else { - throw new Error(`Invalid Option: objname must be a string or a buffer, got ${options.objname}`); - } - if(options.objname !== undefined){ - if(typeof options.objname === 'number'){ - if(options.columns !== false){ - throw Error('Invalid Option: objname index cannot be combined with columns or be defined as a field'); - } - }else { // A string or a buffer - if(options.columns === false){ - throw Error('Invalid Option: objname field must be combined with columns or be defined as an index'); - } - } - } - // Normalize option `on_record` - if(options.on_record === undefined || options.on_record === null){ - options.on_record = undefined; - }else if(typeof options.on_record !== 'function'){ - throw new CsvError$1('CSV_INVALID_OPTION_ON_RECORD', [ - 'Invalid option `on_record`:', - 'expect a function,', - `got ${JSON.stringify(options.on_record)}` + options.record_delimiter = [options.record_delimiter]; + }else if(!Array.isArray(options.record_delimiter)){ + throw new CsvError$1('CSV_INVALID_OPTION_RECORD_DELIMITER', [ + 'Invalid option `record_delimiter`:', + 'value must be a string, a buffer or array of string|buffer,', + `got ${JSON.stringify(options.record_delimiter)}` + ], options); + } + options.record_delimiter = options.record_delimiter.map(function(rd, i){ + if(typeof rd !== 'string' && ! Buffer.isBuffer(rd)){ + throw new CsvError$1('CSV_INVALID_OPTION_RECORD_DELIMITER', [ + 'Invalid option `record_delimiter`:', + 'value must be a string, a buffer or array of string|buffer', + `at index ${i},`, + `got ${JSON.stringify(rd)}` ], options); - } - // Normalize option `quote` - if(options.quote === null || options.quote === false || options.quote === ''){ - options.quote = null; - }else { - if(options.quote === undefined || options.quote === true){ - options.quote = Buffer.from('"', options.encoding); - }else if(typeof options.quote === 'string'){ - options.quote = Buffer.from(options.quote, options.encoding); - } - if(!Buffer.isBuffer(options.quote)){ - throw new Error(`Invalid Option: quote must be a buffer or a string, got ${JSON.stringify(options.quote)}`); - } - } - // Normalize option `raw` - if(options.raw === undefined || options.raw === null || options.raw === false){ - options.raw = false; - }else if(options.raw !== true){ - throw new Error(`Invalid Option: raw must be true, got ${JSON.stringify(options.raw)}`); - } - // Normalize option `record_delimiter` - if(options.record_delimiter === undefined){ - options.record_delimiter = []; - }else if(typeof options.record_delimiter === 'string' || Buffer.isBuffer(options.record_delimiter)){ - if(options.record_delimiter.length === 0){ - throw new CsvError$1('CSV_INVALID_OPTION_RECORD_DELIMITER', [ - 'Invalid option `record_delimiter`:', - 'value must be a non empty string or buffer,', - `got ${JSON.stringify(options.record_delimiter)}` - ], options); - } - options.record_delimiter = [options.record_delimiter]; - }else if(!Array.isArray(options.record_delimiter)){ + }else if(rd.length === 0){ throw new CsvError$1('CSV_INVALID_OPTION_RECORD_DELIMITER', [ 'Invalid option `record_delimiter`:', - 'value must be a string, a buffer or array of string|buffer,', - `got ${JSON.stringify(options.record_delimiter)}` + 'value must be a non empty string or buffer', + `at index ${i},`, + `got ${JSON.stringify(rd)}` ], options); } - options.record_delimiter = options.record_delimiter.map(function(rd, i){ - if(typeof rd !== 'string' && ! Buffer.isBuffer(rd)){ - throw new CsvError$1('CSV_INVALID_OPTION_RECORD_DELIMITER', [ - 'Invalid option `record_delimiter`:', - 'value must be a string, a buffer or array of string|buffer', - `at index ${i},`, - `got ${JSON.stringify(rd)}` - ], options); - }else if(rd.length === 0){ - throw new CsvError$1('CSV_INVALID_OPTION_RECORD_DELIMITER', [ - 'Invalid option `record_delimiter`:', - 'value must be a non empty string or buffer', - `at index ${i},`, - `got ${JSON.stringify(rd)}` - ], options); - } - if(typeof rd === 'string'){ - rd = Buffer.from(rd, options.encoding); - } - return rd; - }); - // Normalize option `relax_column_count` - if(typeof options.relax_column_count === 'boolean');else if(options.relax_column_count === undefined || options.relax_column_count === null){ - options.relax_column_count = false; - }else { - throw new Error(`Invalid Option: relax_column_count must be a boolean, got ${JSON.stringify(options.relax_column_count)}`); - } - if(typeof options.relax_column_count_less === 'boolean');else if(options.relax_column_count_less === undefined || options.relax_column_count_less === null){ - options.relax_column_count_less = false; - }else { - throw new Error(`Invalid Option: relax_column_count_less must be a boolean, got ${JSON.stringify(options.relax_column_count_less)}`); - } - if(typeof options.relax_column_count_more === 'boolean');else if(options.relax_column_count_more === undefined || options.relax_column_count_more === null){ - options.relax_column_count_more = false; - }else { - throw new Error(`Invalid Option: relax_column_count_more must be a boolean, got ${JSON.stringify(options.relax_column_count_more)}`); - } - // Normalize option `relax_quotes` - if(typeof options.relax_quotes === 'boolean');else if(options.relax_quotes === undefined || options.relax_quotes === null){ - options.relax_quotes = false; - }else { - throw new Error(`Invalid Option: relax_quotes must be a boolean, got ${JSON.stringify(options.relax_quotes)}`); - } - // Normalize option `skip_empty_lines` - if(typeof options.skip_empty_lines === 'boolean');else if(options.skip_empty_lines === undefined || options.skip_empty_lines === null){ - options.skip_empty_lines = false; - }else { - throw new Error(`Invalid Option: skip_empty_lines must be a boolean, got ${JSON.stringify(options.skip_empty_lines)}`); + if(typeof rd === 'string'){ + rd = Buffer.from(rd, options.encoding); } - // Normalize option `skip_records_with_empty_values` - if(typeof options.skip_records_with_empty_values === 'boolean');else if(options.skip_records_with_empty_values === undefined || options.skip_records_with_empty_values === null){ - options.skip_records_with_empty_values = false; - }else { - throw new Error(`Invalid Option: skip_records_with_empty_values must be a boolean, got ${JSON.stringify(options.skip_records_with_empty_values)}`); + return rd; + }); + // Normalize option `relax_column_count` + if(typeof options.relax_column_count === 'boolean');else if(options.relax_column_count === undefined || options.relax_column_count === null){ + options.relax_column_count = false; + }else { + throw new Error(`Invalid Option: relax_column_count must be a boolean, got ${JSON.stringify(options.relax_column_count)}`); + } + if(typeof options.relax_column_count_less === 'boolean');else if(options.relax_column_count_less === undefined || options.relax_column_count_less === null){ + options.relax_column_count_less = false; + }else { + throw new Error(`Invalid Option: relax_column_count_less must be a boolean, got ${JSON.stringify(options.relax_column_count_less)}`); + } + if(typeof options.relax_column_count_more === 'boolean');else if(options.relax_column_count_more === undefined || options.relax_column_count_more === null){ + options.relax_column_count_more = false; + }else { + throw new Error(`Invalid Option: relax_column_count_more must be a boolean, got ${JSON.stringify(options.relax_column_count_more)}`); + } + // Normalize option `relax_quotes` + if(typeof options.relax_quotes === 'boolean');else if(options.relax_quotes === undefined || options.relax_quotes === null){ + options.relax_quotes = false; + }else { + throw new Error(`Invalid Option: relax_quotes must be a boolean, got ${JSON.stringify(options.relax_quotes)}`); + } + // Normalize option `skip_empty_lines` + if(typeof options.skip_empty_lines === 'boolean');else if(options.skip_empty_lines === undefined || options.skip_empty_lines === null){ + options.skip_empty_lines = false; + }else { + throw new Error(`Invalid Option: skip_empty_lines must be a boolean, got ${JSON.stringify(options.skip_empty_lines)}`); + } + // Normalize option `skip_records_with_empty_values` + if(typeof options.skip_records_with_empty_values === 'boolean');else if(options.skip_records_with_empty_values === undefined || options.skip_records_with_empty_values === null){ + options.skip_records_with_empty_values = false; + }else { + throw new Error(`Invalid Option: skip_records_with_empty_values must be a boolean, got ${JSON.stringify(options.skip_records_with_empty_values)}`); + } + // Normalize option `skip_records_with_error` + if(typeof options.skip_records_with_error === 'boolean');else if(options.skip_records_with_error === undefined || options.skip_records_with_error === null){ + options.skip_records_with_error = false; + }else { + throw new Error(`Invalid Option: skip_records_with_error must be a boolean, got ${JSON.stringify(options.skip_records_with_error)}`); + } + // Normalize option `rtrim` + if(options.rtrim === undefined || options.rtrim === null || options.rtrim === false){ + options.rtrim = false; + }else if(options.rtrim !== true){ + throw new Error(`Invalid Option: rtrim must be a boolean, got ${JSON.stringify(options.rtrim)}`); + } + // Normalize option `ltrim` + if(options.ltrim === undefined || options.ltrim === null || options.ltrim === false){ + options.ltrim = false; + }else if(options.ltrim !== true){ + throw new Error(`Invalid Option: ltrim must be a boolean, got ${JSON.stringify(options.ltrim)}`); + } + // Normalize option `trim` + if(options.trim === undefined || options.trim === null || options.trim === false){ + options.trim = false; + }else if(options.trim !== true){ + throw new Error(`Invalid Option: trim must be a boolean, got ${JSON.stringify(options.trim)}`); + } + // Normalize options `trim`, `ltrim` and `rtrim` + if(options.trim === true && opts.ltrim !== false){ + options.ltrim = true; + }else if(options.ltrim !== true){ + options.ltrim = false; + } + if(options.trim === true && opts.rtrim !== false){ + options.rtrim = true; + }else if(options.rtrim !== true){ + options.rtrim = false; + } + // Normalize option `to` + if(options.to === undefined || options.to === null){ + options.to = -1; + }else { + if(typeof options.to === 'string' && /\d+/.test(options.to)){ + options.to = parseInt(options.to); } - // Normalize option `skip_records_with_error` - if(typeof options.skip_records_with_error === 'boolean');else if(options.skip_records_with_error === undefined || options.skip_records_with_error === null){ - options.skip_records_with_error = false; - }else { - throw new Error(`Invalid Option: skip_records_with_error must be a boolean, got ${JSON.stringify(options.skip_records_with_error)}`); - } - // Normalize option `rtrim` - if(options.rtrim === undefined || options.rtrim === null || options.rtrim === false){ - options.rtrim = false; - }else if(options.rtrim !== true){ - throw new Error(`Invalid Option: rtrim must be a boolean, got ${JSON.stringify(options.rtrim)}`); - } - // Normalize option `ltrim` - if(options.ltrim === undefined || options.ltrim === null || options.ltrim === false){ - options.ltrim = false; - }else if(options.ltrim !== true){ - throw new Error(`Invalid Option: ltrim must be a boolean, got ${JSON.stringify(options.ltrim)}`); - } - // Normalize option `trim` - if(options.trim === undefined || options.trim === null || options.trim === false){ - options.trim = false; - }else if(options.trim !== true){ - throw new Error(`Invalid Option: trim must be a boolean, got ${JSON.stringify(options.trim)}`); - } - // Normalize options `trim`, `ltrim` and `rtrim` - if(options.trim === true && opts.ltrim !== false){ - options.ltrim = true; - }else if(options.ltrim !== true){ - options.ltrim = false; - } - if(options.trim === true && opts.rtrim !== false){ - options.rtrim = true; - }else if(options.rtrim !== true){ - options.rtrim = false; - } - // Normalize option `to` - if(options.to === undefined || options.to === null){ - options.to = -1; - }else { - if(typeof options.to === 'string' && /\d+/.test(options.to)){ - options.to = parseInt(options.to); - } - if(Number.isInteger(options.to)){ - if(options.to <= 0){ - throw new Error(`Invalid Option: to must be a positive integer greater than 0, got ${JSON.stringify(opts.to)}`); - } - }else { - throw new Error(`Invalid Option: to must be an integer, got ${JSON.stringify(opts.to)}`); + if(Number.isInteger(options.to)){ + if(options.to <= 0){ + throw new Error(`Invalid Option: to must be a positive integer greater than 0, got ${JSON.stringify(opts.to)}`); } - } - // Normalize option `to_line` - if(options.to_line === undefined || options.to_line === null){ - options.to_line = -1; }else { - if(typeof options.to_line === 'string' && /\d+/.test(options.to_line)){ - options.to_line = parseInt(options.to_line); - } - if(Number.isInteger(options.to_line)){ - if(options.to_line <= 0){ - throw new Error(`Invalid Option: to_line must be a positive integer greater than 0, got ${JSON.stringify(opts.to_line)}`); - } - }else { - throw new Error(`Invalid Option: to_line must be an integer, got ${JSON.stringify(opts.to_line)}`); - } + throw new Error(`Invalid Option: to must be an integer, got ${JSON.stringify(opts.to)}`); } - this.info = { - bytes: 0, - comment_lines: 0, - empty_lines: 0, - invalid_field_length: 0, - lines: 1, - records: 0 - }; - this.options = options; - this.state = { - bomSkipped: false, - bufBytesStart: 0, - castField: fnCastField, - commenting: false, - // Current error encountered by a record - error: undefined, - enabled: options.from_line === 1, - escaping: false, - escapeIsQuote: Buffer.isBuffer(options.escape) && Buffer.isBuffer(options.quote) && Buffer.compare(options.escape, options.quote) === 0, - // columns can be `false`, `true`, `Array` - expectedRecordLength: Array.isArray(options.columns) ? options.columns.length : undefined, - field: new ResizeableBuffer(20), - firstLineToHeaders: fnFirstLineToHeaders, - needMoreDataSize: Math.max( - // Skip if the remaining buffer smaller than comment - options.comment !== null ? options.comment.length : 0, - // Skip if the remaining buffer can be delimiter - ...options.delimiter.map((delimiter) => delimiter.length), - // Skip if the remaining buffer can be escape sequence - options.quote !== null ? options.quote.length : 0, - ), - previousBuf: undefined, - quoting: false, - stop: false, - rawBuffer: new ResizeableBuffer(100), - record: [], - recordHasError: false, - record_length: 0, - recordDelimiterMaxLength: options.record_delimiter.length === 0 ? 2 : Math.max(...options.record_delimiter.map((v) => v.length)), - trimChars: [Buffer.from(' ', options.encoding)[0], Buffer.from('\t', options.encoding)[0]], - wasQuoting: false, - wasRowDelimiter: false - }; - } - // Implementation of `Transform._transform` - _transform(buf, encoding, callback){ - if(this.state.stop === true){ - return; - } - const err = this.__parse(buf, false); - if(err !== undefined){ - this.state.stop = true; - } - callback(err); } - // Implementation of `Transform._flush` - _flush(callback){ - if(this.state.stop === true){ - return; + // Normalize option `to_line` + if(options.to_line === undefined || options.to_line === null){ + options.to_line = -1; + }else { + if(typeof options.to_line === 'string' && /\d+/.test(options.to_line)){ + options.to_line = parseInt(options.to_line); } - const err = this.__parse(undefined, true); - callback(err); - } - // Central parser implementation - __parse(nextBuf, end){ - const {bom, comment, escape, from_line, ltrim, max_record_size, quote, raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options; - let {record_delimiter} = this.options; - const {bomSkipped, previousBuf, rawBuffer, escapeIsQuote} = this.state; - let buf; - if(previousBuf === undefined){ - if(nextBuf === undefined){ - // Handle empty string - this.push(null); - return; - }else { - buf = nextBuf; + if(Number.isInteger(options.to_line)){ + if(options.to_line <= 0){ + throw new Error(`Invalid Option: to_line must be a positive integer greater than 0, got ${JSON.stringify(opts.to_line)}`); } - }else if(previousBuf !== undefined && nextBuf === undefined){ - buf = previousBuf; }else { - buf = Buffer.concat([previousBuf, nextBuf]); - } - // Handle UTF BOM - if(bomSkipped === false){ - if(bom === false){ - this.state.bomSkipped = true; - }else if(buf.length < 3){ - // No enough data - if(end === false){ - // Wait for more data - this.state.previousBuf = buf; + throw new Error(`Invalid Option: to_line must be an integer, got ${JSON.stringify(opts.to_line)}`); + } + } + return options; +}; + +const isRecordEmpty = function(record){ + return record.every((field) => field == null || field.toString && field.toString().trim() === ''); +}; + +// white space characters +// https://en.wikipedia.org/wiki/Whitespace_character +// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions/Character_Classes#Types +// \f\n\r\t\v\u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff +const tab = 9; +const nl = 10; // \n, 0x0A in hexadecimal, 10 in decimal +const np = 12; +const cr = 13; // \r, 0x0D in hexadécimal, 13 in decimal +const space = 32; +const boms = { + // Note, the following are equals: + // Buffer.from("\ufeff") + // Buffer.from([239, 187, 191]) + // Buffer.from('EFBBBF', 'hex') + 'utf8': Buffer.from([239, 187, 191]), + // Note, the following are equals: + // Buffer.from "\ufeff", 'utf16le + // Buffer.from([255, 254]) + 'utf16le': Buffer.from([255, 254]) +}; + +const transform$1 = function(original_options = {}) { + const info = { + bytes: 0, + comment_lines: 0, + empty_lines: 0, + invalid_field_length: 0, + lines: 1, + records: 0 + }; + const options = normalize_options$1(original_options); + return { + info: info, + original_options: original_options, + options: options, + state: init_state(options), + __needMoreData: function(i, bufLen, end){ + if(end) return false; + const {quote} = this.options; + const {quoting, needMoreDataSize, recordDelimiterMaxLength} = this.state; + const numOfCharLeft = bufLen - i - 1; + const requiredLength = Math.max( + needMoreDataSize, + // Skip if the remaining buffer smaller than record delimiter + recordDelimiterMaxLength, + // Skip if the remaining buffer can be record delimiter following the closing quote + // 1 is for quote.length + quoting ? (quote.length + recordDelimiterMaxLength) : 0, + ); + return numOfCharLeft < requiredLength; + }, + // Central parser implementation + parse: function(nextBuf, end, push, close){ + const {bom, comment, escape, from_line, ltrim, max_record_size, quote, raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options; + let {record_delimiter} = this.options; + const {bomSkipped, previousBuf, rawBuffer, escapeIsQuote} = this.state; + let buf; + if(previousBuf === undefined){ + if(nextBuf === undefined){ + // Handle empty string + close(); return; + }else { + buf = nextBuf; } + }else if(previousBuf !== undefined && nextBuf === undefined){ + buf = previousBuf; }else { - for(const encoding in boms){ - if(boms[encoding].compare(buf, 0, boms[encoding].length) === 0){ - // Skip BOM - const bomLength = boms[encoding].length; - this.state.bufBytesStart += bomLength; - buf = buf.slice(bomLength); - // Renormalize original options with the new encoding - this.__normalizeOptions({...this.__originalOptions, encoding: encoding}); - break; + buf = Buffer.concat([previousBuf, nextBuf]); + } + // Handle UTF BOM + if(bomSkipped === false){ + if(bom === false){ + this.state.bomSkipped = true; + }else if(buf.length < 3){ + // No enough data + if(end === false){ + // Wait for more data + this.state.previousBuf = buf; + return; } + }else { + for(const encoding in boms){ + if(boms[encoding].compare(buf, 0, boms[encoding].length) === 0){ + // Skip BOM + const bomLength = boms[encoding].length; + this.state.bufBytesStart += bomLength; + buf = buf.slice(bomLength); + // Renormalize original options with the new encoding + this.options = normalize_options$1({...this.original_options, encoding: encoding}); + break; + } + } + this.state.bomSkipped = true; } - this.state.bomSkipped = true; } - } - const bufLen = buf.length; - let pos; - for(pos = 0; pos < bufLen; pos++){ - // Ensure we get enough space to look ahead - // There should be a way to move this out of the loop - if(this.__needMoreData(pos, bufLen, end)){ - break; - } - if(this.state.wasRowDelimiter === true){ - this.info.lines++; - this.state.wasRowDelimiter = false; - } - if(to_line !== -1 && this.info.lines > to_line){ - this.state.stop = true; - this.push(null); - return; - } - // Auto discovery of record_delimiter, unix, mac and windows supported - if(this.state.quoting === false && record_delimiter.length === 0){ - const record_delimiterCount = this.__autoDiscoverRecordDelimiter(buf, pos); - if(record_delimiterCount){ - record_delimiter = this.options.record_delimiter; + const bufLen = buf.length; + let pos; + for(pos = 0; pos < bufLen; pos++){ + // Ensure we get enough space to look ahead + // There should be a way to move this out of the loop + if(this.__needMoreData(pos, bufLen, end)){ + break; } - } - const chr = buf[pos]; - if(raw === true){ - rawBuffer.append(chr); - } - if((chr === cr || chr === nl) && this.state.wasRowDelimiter === false){ - this.state.wasRowDelimiter = true; - } - // Previous char was a valid escape char - // treat the current char as a regular char - if(this.state.escaping === true){ - this.state.escaping = false; - }else { - // Escape is only active inside quoted fields - // We are quoting, the char is an escape chr and there is a chr to escape - // if(escape !== null && this.state.quoting === true && chr === escape && pos + 1 < bufLen){ - if(escape !== null && this.state.quoting === true && this.__isEscape(buf, pos, chr) && pos + escape.length < bufLen){ - if(escapeIsQuote){ - if(this.__isQuote(buf, pos+escape.length)){ - this.state.escaping = true; - pos += escape.length - 1; - continue; - } - }else { - this.state.escaping = true; - pos += escape.length - 1; - continue; + if(this.state.wasRowDelimiter === true){ + this.info.lines++; + this.state.wasRowDelimiter = false; + } + if(to_line !== -1 && this.info.lines > to_line){ + this.state.stop = true; + close(); + return; + } + // Auto discovery of record_delimiter, unix, mac and windows supported + if(this.state.quoting === false && record_delimiter.length === 0){ + const record_delimiterCount = this.__autoDiscoverRecordDelimiter(buf, pos); + if(record_delimiterCount){ + record_delimiter = this.options.record_delimiter; } } - // Not currently escaping and chr is a quote - // TODO: need to compare bytes instead of single char - if(this.state.commenting === false && this.__isQuote(buf, pos)){ - if(this.state.quoting === true){ - const nextChr = buf[pos+quote.length]; - const isNextChrTrimable = rtrim && this.__isCharTrimable(nextChr); - const isNextChrComment = comment !== null && this.__compareBytes(comment, buf, pos+quote.length, nextChr); - const isNextChrDelimiter = this.__isDelimiter(buf, pos+quote.length, nextChr); - const isNextChrRecordDelimiter = record_delimiter.length === 0 ? this.__autoDiscoverRecordDelimiter(buf, pos+quote.length) : this.__isRecordDelimiter(nextChr, buf, pos+quote.length); - // Escape a quote - // Treat next char as a regular character - if(escape !== null && this.__isEscape(buf, pos, chr) && this.__isQuote(buf, pos + escape.length)){ + const chr = buf[pos]; + if(raw === true){ + rawBuffer.append(chr); + } + if((chr === cr || chr === nl) && this.state.wasRowDelimiter === false){ + this.state.wasRowDelimiter = true; + } + // Previous char was a valid escape char + // treat the current char as a regular char + if(this.state.escaping === true){ + this.state.escaping = false; + }else { + // Escape is only active inside quoted fields + // We are quoting, the char is an escape chr and there is a chr to escape + // if(escape !== null && this.state.quoting === true && chr === escape && pos + 1 < bufLen){ + if(escape !== null && this.state.quoting === true && this.__isEscape(buf, pos, chr) && pos + escape.length < bufLen){ + if(escapeIsQuote){ + if(this.__isQuote(buf, pos+escape.length)){ + this.state.escaping = true; + pos += escape.length - 1; + continue; + } + }else { + this.state.escaping = true; pos += escape.length - 1; - }else if(!nextChr || isNextChrDelimiter || isNextChrRecordDelimiter || isNextChrComment || isNextChrTrimable){ - this.state.quoting = false; - this.state.wasQuoting = true; - pos += quote.length - 1; continue; - }else if(relax_quotes === false){ - const err = this.__error( - new CsvError$1('CSV_INVALID_CLOSING_QUOTE', [ - 'Invalid Closing Quote:', - `got "${String.fromCharCode(nextChr)}"`, - `at line ${this.info.lines}`, - 'instead of delimiter, record delimiter, trimable character', - '(if activated) or comment', - ], this.options, this.__infoField()) - ); - if(err !== undefined) return err; - }else { - this.state.quoting = false; - this.state.wasQuoting = true; - this.state.field.prepend(quote); - pos += quote.length - 1; } - }else { - if(this.state.field.length !== 0){ - // In relax_quotes mode, treat opening quote preceded by chrs as regular - if(relax_quotes === false){ + } + // Not currently escaping and chr is a quote + // TODO: need to compare bytes instead of single char + if(this.state.commenting === false && this.__isQuote(buf, pos)){ + if(this.state.quoting === true){ + const nextChr = buf[pos+quote.length]; + const isNextChrTrimable = rtrim && this.__isCharTrimable(nextChr); + const isNextChrComment = comment !== null && this.__compareBytes(comment, buf, pos+quote.length, nextChr); + const isNextChrDelimiter = this.__isDelimiter(buf, pos+quote.length, nextChr); + const isNextChrRecordDelimiter = record_delimiter.length === 0 ? this.__autoDiscoverRecordDelimiter(buf, pos+quote.length) : this.__isRecordDelimiter(nextChr, buf, pos+quote.length); + // Escape a quote + // Treat next char as a regular character + if(escape !== null && this.__isEscape(buf, pos, chr) && this.__isQuote(buf, pos + escape.length)){ + pos += escape.length - 1; + }else if(!nextChr || isNextChrDelimiter || isNextChrRecordDelimiter || isNextChrComment || isNextChrTrimable){ + this.state.quoting = false; + this.state.wasQuoting = true; + pos += quote.length - 1; + continue; + }else if(relax_quotes === false){ const err = this.__error( - new CsvError$1('INVALID_OPENING_QUOTE', [ - 'Invalid Opening Quote:', - `a quote is found inside a field at line ${this.info.lines}`, - ], this.options, this.__infoField(), { - field: this.state.field, - }) + new CsvError$1('CSV_INVALID_CLOSING_QUOTE', [ + 'Invalid Closing Quote:', + `got "${String.fromCharCode(nextChr)}"`, + `at line ${this.info.lines}`, + 'instead of delimiter, record delimiter, trimable character', + '(if activated) or comment', + ], this.options, this.__infoField()) ); if(err !== undefined) return err; + }else { + this.state.quoting = false; + this.state.wasQuoting = true; + this.state.field.prepend(quote); + pos += quote.length - 1; } }else { - this.state.quoting = true; - pos += quote.length - 1; - continue; - } - } - } - if(this.state.quoting === false){ - const recordDelimiterLength = this.__isRecordDelimiter(chr, buf, pos); - if(recordDelimiterLength !== 0){ - // Do not emit comments which take a full line - const skipCommentLine = this.state.commenting && (this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0); - if(skipCommentLine){ - this.info.comment_lines++; - // Skip full comment line - }else { - // Activate records emition if above from_line - if(this.state.enabled === false && this.info.lines + (this.state.wasRowDelimiter === true ? 1: 0) >= from_line){ - this.state.enabled = true; - this.__resetField(); - this.__resetRecord(); - pos += recordDelimiterLength - 1; + if(this.state.field.length !== 0){ + // In relax_quotes mode, treat opening quote preceded by chrs as regular + if(relax_quotes === false){ + const err = this.__error( + new CsvError$1('INVALID_OPENING_QUOTE', [ + 'Invalid Opening Quote:', + `a quote is found inside a field at line ${this.info.lines}`, + ], this.options, this.__infoField(), { + field: this.state.field, + }) + ); + if(err !== undefined) return err; + } + }else { + this.state.quoting = true; + pos += quote.length - 1; continue; } - // Skip if line is empty and skip_empty_lines activated - if(skip_empty_lines === true && this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0){ - this.info.empty_lines++; - pos += recordDelimiterLength - 1; - continue; + } + } + if(this.state.quoting === false){ + const recordDelimiterLength = this.__isRecordDelimiter(chr, buf, pos); + if(recordDelimiterLength !== 0){ + // Do not emit comments which take a full line + const skipCommentLine = this.state.commenting && (this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0); + if(skipCommentLine){ + this.info.comment_lines++; + // Skip full comment line + }else { + // Activate records emition if above from_line + if(this.state.enabled === false && this.info.lines + (this.state.wasRowDelimiter === true ? 1: 0) >= from_line){ + this.state.enabled = true; + this.__resetField(); + this.__resetRecord(); + pos += recordDelimiterLength - 1; + continue; + } + // Skip if line is empty and skip_empty_lines activated + if(skip_empty_lines === true && this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0){ + this.info.empty_lines++; + pos += recordDelimiterLength - 1; + continue; + } + this.info.bytes = this.state.bufBytesStart + pos; + const errField = this.__onField(); + if(errField !== undefined) return errField; + this.info.bytes = this.state.bufBytesStart + pos + recordDelimiterLength; + const errRecord = this.__onRecord(push); + if(errRecord !== undefined) return errRecord; + if(to !== -1 && this.info.records >= to){ + this.state.stop = true; + close(); + return; + } } + this.state.commenting = false; + pos += recordDelimiterLength - 1; + continue; + } + if(this.state.commenting){ + continue; + } + const commentCount = comment === null ? 0 : this.__compareBytes(comment, buf, pos, chr); + if(commentCount !== 0){ + this.state.commenting = true; + continue; + } + const delimiterLength = this.__isDelimiter(buf, pos, chr); + if(delimiterLength !== 0){ this.info.bytes = this.state.bufBytesStart + pos; const errField = this.__onField(); if(errField !== undefined) return errField; - this.info.bytes = this.state.bufBytesStart + pos + recordDelimiterLength; - const errRecord = this.__onRecord(); - if(errRecord !== undefined) return errRecord; - if(to !== -1 && this.info.records >= to){ - this.state.stop = true; - this.push(null); - return; - } + pos += delimiterLength - 1; + continue; } - this.state.commenting = false; - pos += recordDelimiterLength - 1; - continue; } - if(this.state.commenting){ - continue; - } - const commentCount = comment === null ? 0 : this.__compareBytes(comment, buf, pos, chr); - if(commentCount !== 0){ - this.state.commenting = true; - continue; - } - const delimiterLength = this.__isDelimiter(buf, pos, chr); - if(delimiterLength !== 0){ - this.info.bytes = this.state.bufBytesStart + pos; - const errField = this.__onField(); - if(errField !== undefined) return errField; - pos += delimiterLength - 1; - continue; + } + if(this.state.commenting === false){ + if(max_record_size !== 0 && this.state.record_length + this.state.field.length > max_record_size){ + const err = this.__error( + new CsvError$1('CSV_MAX_RECORD_SIZE', [ + 'Max Record Size:', + 'record exceed the maximum number of tolerated bytes', + `of ${max_record_size}`, + `at line ${this.info.lines}`, + ], this.options, this.__infoField()) + ); + if(err !== undefined) return err; } } - } - if(this.state.commenting === false){ - if(max_record_size !== 0 && this.state.record_length + this.state.field.length > max_record_size){ + const lappend = ltrim === false || this.state.quoting === true || this.state.field.length !== 0 || !this.__isCharTrimable(chr); + // rtrim in non quoting is handle in __onField + const rappend = rtrim === false || this.state.wasQuoting === false; + if(lappend === true && rappend === true){ + this.state.field.append(chr); + }else if(rtrim === true && !this.__isCharTrimable(chr)){ const err = this.__error( - new CsvError$1('CSV_MAX_RECORD_SIZE', [ - 'Max Record Size:', - 'record exceed the maximum number of tolerated bytes', - `of ${max_record_size}`, + new CsvError$1('CSV_NON_TRIMABLE_CHAR_AFTER_CLOSING_QUOTE', [ + 'Invalid Closing Quote:', + 'found non trimable byte after quote', `at line ${this.info.lines}`, ], this.options, this.__infoField()) ); if(err !== undefined) return err; } } - const lappend = ltrim === false || this.state.quoting === true || this.state.field.length !== 0 || !this.__isCharTrimable(chr); - // rtrim in non quoting is handle in __onField - const rappend = rtrim === false || this.state.wasQuoting === false; - if(lappend === true && rappend === true){ - this.state.field.append(chr); - }else if(rtrim === true && !this.__isCharTrimable(chr)){ - const err = this.__error( - new CsvError$1('CSV_NON_TRIMABLE_CHAR_AFTER_CLOSING_QUOTE', [ - 'Invalid Closing Quote:', - 'found non trimable byte after quote', - `at line ${this.info.lines}`, - ], this.options, this.__infoField()) - ); - if(err !== undefined) return err; - } - } - if(end === true){ - // Ensure we are not ending in a quoting state - if(this.state.quoting === true){ - const err = this.__error( - new CsvError$1('CSV_QUOTE_NOT_CLOSED', [ - 'Quote Not Closed:', - `the parsing is finished with an opening quote at line ${this.info.lines}`, - ], this.options, this.__infoField()) - ); - if(err !== undefined) return err; + if(end === true){ + // Ensure we are not ending in a quoting state + if(this.state.quoting === true){ + const err = this.__error( + new CsvError$1('CSV_QUOTE_NOT_CLOSED', [ + 'Quote Not Closed:', + `the parsing is finished with an opening quote at line ${this.info.lines}`, + ], this.options, this.__infoField()) + ); + if(err !== undefined) return err; + }else { + // Skip last line if it has no characters + if(this.state.wasQuoting === true || this.state.record.length !== 0 || this.state.field.length !== 0){ + this.info.bytes = this.state.bufBytesStart + pos; + const errField = this.__onField(); + if(errField !== undefined) return errField; + const errRecord = this.__onRecord(push); + if(errRecord !== undefined) return errRecord; + }else if(this.state.wasRowDelimiter === true){ + this.info.empty_lines++; + }else if(this.state.commenting === true){ + this.info.comment_lines++; + } + } }else { - // Skip last line if it has no characters - if(this.state.wasQuoting === true || this.state.record.length !== 0 || this.state.field.length !== 0){ - this.info.bytes = this.state.bufBytesStart + pos; - const errField = this.__onField(); - if(errField !== undefined) return errField; - const errRecord = this.__onRecord(); - if(errRecord !== undefined) return errRecord; - }else if(this.state.wasRowDelimiter === true){ - this.info.empty_lines++; - }else if(this.state.commenting === true){ - this.info.comment_lines++; + this.state.bufBytesStart += pos; + this.state.previousBuf = buf.slice(pos); + } + if(this.state.wasRowDelimiter === true){ + this.info.lines++; + this.state.wasRowDelimiter = false; + } + }, + __onRecord: function(push){ + const {columns, group_columns_by_name, encoding, info, from, relax_column_count, relax_column_count_less, relax_column_count_more, raw, skip_records_with_empty_values} = this.options; + const {enabled, record} = this.state; + if(enabled === false){ + return this.__resetRecord(); + } + // Convert the first line into column names + const recordLength = record.length; + if(columns === true){ + if(skip_records_with_empty_values === true && isRecordEmpty(record)){ + this.__resetRecord(); + return; + } + return this.__firstLineToColumns(record); + } + if(columns === false && this.info.records === 0){ + this.state.expectedRecordLength = recordLength; + } + if(recordLength !== this.state.expectedRecordLength){ + const err = columns === false ? + new CsvError$1('CSV_RECORD_INCONSISTENT_FIELDS_LENGTH', [ + 'Invalid Record Length:', + `expect ${this.state.expectedRecordLength},`, + `got ${recordLength} on line ${this.info.lines}`, + ], this.options, this.__infoField(), { + record: record, + }) + : + new CsvError$1('CSV_RECORD_INCONSISTENT_COLUMNS', [ + 'Invalid Record Length:', + `columns length is ${columns.length},`, // rename columns + `got ${recordLength} on line ${this.info.lines}`, + ], this.options, this.__infoField(), { + record: record, + }); + if(relax_column_count === true || + (relax_column_count_less === true && recordLength < this.state.expectedRecordLength) || + (relax_column_count_more === true && recordLength > this.state.expectedRecordLength)){ + this.info.invalid_field_length++; + this.state.error = err; + // Error is undefined with skip_records_with_error + }else { + const finalErr = this.__error(err); + if(finalErr) return finalErr; } } - }else { - this.state.bufBytesStart += pos; - this.state.previousBuf = buf.slice(pos); - } - if(this.state.wasRowDelimiter === true){ - this.info.lines++; - this.state.wasRowDelimiter = false; - } - } - __onRecord(){ - const {columns, group_columns_by_name, encoding, info, from, relax_column_count, relax_column_count_less, relax_column_count_more, raw, skip_records_with_empty_values} = this.options; - const {enabled, record} = this.state; - if(enabled === false){ - return this.__resetRecord(); - } - // Convert the first line into column names - const recordLength = record.length; - if(columns === true){ if(skip_records_with_empty_values === true && isRecordEmpty(record)){ this.__resetRecord(); return; } - return this.__firstLineToColumns(record); - } - if(columns === false && this.info.records === 0){ - this.state.expectedRecordLength = recordLength; - } - if(recordLength !== this.state.expectedRecordLength){ - const err = columns === false ? - new CsvError$1('CSV_RECORD_INCONSISTENT_FIELDS_LENGTH', [ - 'Invalid Record Length:', - `expect ${this.state.expectedRecordLength},`, - `got ${recordLength} on line ${this.info.lines}`, - ], this.options, this.__infoField(), { - record: record, - }) - : - new CsvError$1('CSV_RECORD_INCONSISTENT_COLUMNS', [ - 'Invalid Record Length:', - `columns length is ${columns.length},`, // rename columns - `got ${recordLength} on line ${this.info.lines}`, - ], this.options, this.__infoField(), { - record: record, - }); - if(relax_column_count === true || - (relax_column_count_less === true && recordLength < this.state.expectedRecordLength) || - (relax_column_count_more === true && recordLength > this.state.expectedRecordLength)){ - this.info.invalid_field_length++; - this.state.error = err; - // Error is undefined with skip_records_with_error - }else { - const finalErr = this.__error(err); - if(finalErr) return finalErr; + if(this.state.recordHasError === true){ + this.__resetRecord(); + this.state.recordHasError = false; + return; } - } - if(skip_records_with_empty_values === true && isRecordEmpty(record)){ - this.__resetRecord(); - return; - } - if(this.state.recordHasError === true){ - this.__resetRecord(); - this.state.recordHasError = false; - return; - } - this.info.records++; - if(from === 1 || this.info.records >= from){ - const {objname} = this.options; - // With columns, records are object - if(columns !== false){ - const obj = {}; - // Transform record array to an object - for(let i = 0, l = record.length; i < l; i++){ - if(columns[i] === undefined || columns[i].disabled) continue; - // Turn duplicate columns into an array - if (group_columns_by_name === true && obj[columns[i].name] !== undefined) { - if (Array.isArray(obj[columns[i].name])) { - obj[columns[i].name] = obj[columns[i].name].concat(record[i]); + this.info.records++; + if(from === 1 || this.info.records >= from){ + const {objname} = this.options; + // With columns, records are object + if(columns !== false){ + const obj = {}; + // Transform record array to an object + for(let i = 0, l = record.length; i < l; i++){ + if(columns[i] === undefined || columns[i].disabled) continue; + // Turn duplicate columns into an array + if (group_columns_by_name === true && obj[columns[i].name] !== undefined) { + if (Array.isArray(obj[columns[i].name])) { + obj[columns[i].name] = obj[columns[i].name].concat(record[i]); + } else { + obj[columns[i].name] = [obj[columns[i].name], record[i]]; + } } else { - obj[columns[i].name] = [obj[columns[i].name], record[i]]; + obj[columns[i].name] = record[i]; } - } else { - obj[columns[i].name] = record[i]; - } - } - // Without objname (default) - if(raw === true || info === true){ - const extRecord = Object.assign( - {record: obj}, - (raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}), - (info === true ? {info: this.__infoRecord()}: {}) - ); - const err = this.__push( - objname === undefined ? extRecord : [obj[objname], extRecord] - ); - if(err){ - return err; - } - }else { - const err = this.__push( - objname === undefined ? obj : [obj[objname], obj] - ); - if(err){ - return err; } - } - // Without columns, records are array - }else { - if(raw === true || info === true){ - const extRecord = Object.assign( - {record: record}, - raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}, - info === true ? {info: this.__infoRecord()}: {} - ); - const err = this.__push( - objname === undefined ? extRecord : [record[objname], extRecord] - ); - if(err){ - return err; - } - }else { - const err = this.__push( - objname === undefined ? record : [record[objname], record] - ); - if(err){ - return err; - } - } - } - } - this.__resetRecord(); - } - __firstLineToColumns(record){ - const {firstLineToHeaders} = this.state; - try{ - const headers = firstLineToHeaders === undefined ? record : firstLineToHeaders.call(null, record); - if(!Array.isArray(headers)){ - return this.__error( - new CsvError$1('CSV_INVALID_COLUMN_MAPPING', [ - 'Invalid Column Mapping:', - 'expect an array from column function,', - `got ${JSON.stringify(headers)}` - ], this.options, this.__infoField(), { - headers: headers, - }) - ); - } - const normalizedHeaders = normalizeColumnsArray(headers); - this.state.expectedRecordLength = normalizedHeaders.length; - this.options.columns = normalizedHeaders; - this.__resetRecord(); - return; - }catch(err){ - return err; - } - } - __resetRecord(){ - if(this.options.raw === true){ - this.state.rawBuffer.reset(); - } - this.state.error = undefined; - this.state.record = []; - this.state.record_length = 0; - } - __onField(){ - const {cast, encoding, rtrim, max_record_size} = this.options; - const {enabled, wasQuoting} = this.state; - // Short circuit for the from_line options - if(enabled === false){ - return this.__resetField(); - } - let field = this.state.field.toString(encoding); - if(rtrim === true && wasQuoting === false){ - field = field.trimRight(); - } - if(cast === true){ - const [err, f] = this.__cast(field); - if(err !== undefined) return err; - field = f; - } - this.state.record.push(field); - // Increment record length if record size must not exceed a limit - if(max_record_size !== 0 && typeof field === 'string'){ - this.state.record_length += field.length; - } - this.__resetField(); - } - __resetField(){ - this.state.field.reset(); - this.state.wasQuoting = false; - } - __push(record){ - const {on_record} = this.options; - if(on_record !== undefined){ - const info = this.__infoRecord(); + // Without objname (default) + if(raw === true || info === true){ + const extRecord = Object.assign( + {record: obj}, + (raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}), + (info === true ? {info: this.__infoRecord()}: {}) + ); + const err = this.__push( + objname === undefined ? extRecord : [obj[objname], extRecord] + , push); + if(err){ + return err; + } + }else { + const err = this.__push( + objname === undefined ? obj : [obj[objname], obj] + , push); + if(err){ + return err; + } + } + // Without columns, records are array + }else { + if(raw === true || info === true){ + const extRecord = Object.assign( + {record: record}, + raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}, + info === true ? {info: this.__infoRecord()}: {} + ); + const err = this.__push( + objname === undefined ? extRecord : [record[objname], extRecord] + , push); + if(err){ + return err; + } + }else { + const err = this.__push( + objname === undefined ? record : [record[objname], record] + , push); + if(err){ + return err; + } + } + } + } + this.__resetRecord(); + }, + __firstLineToColumns: function(record){ + const {firstLineToHeaders} = this.state; try{ - record = on_record.call(null, record, info); + const headers = firstLineToHeaders === undefined ? record : firstLineToHeaders.call(null, record); + if(!Array.isArray(headers)){ + return this.__error( + new CsvError$1('CSV_INVALID_COLUMN_MAPPING', [ + 'Invalid Column Mapping:', + 'expect an array from column function,', + `got ${JSON.stringify(headers)}` + ], this.options, this.__infoField(), { + headers: headers, + }) + ); + } + const normalizedHeaders = normalize_columns_array(headers); + this.state.expectedRecordLength = normalizedHeaders.length; + this.options.columns = normalizedHeaders; + this.__resetRecord(); + return; }catch(err){ return err; } - if(record === undefined || record === null){ return; } - } - this.push(record); - } - // Return a tuple with the error and the casted value - __cast(field){ - const {columns, relax_column_count} = this.options; - const isColumns = Array.isArray(columns); - // Dont loose time calling cast - // because the final record is an object - // and this field can't be associated to a key present in columns - if(isColumns === true && relax_column_count && this.options.columns.length <= this.state.record.length){ - return [undefined, undefined]; - } - if(this.state.castField !== null){ - try{ + }, + __resetRecord: function(){ + if(this.options.raw === true){ + this.state.rawBuffer.reset(); + } + this.state.error = undefined; + this.state.record = []; + this.state.record_length = 0; + }, + __onField: function(){ + const {cast, encoding, rtrim, max_record_size} = this.options; + const {enabled, wasQuoting} = this.state; + // Short circuit for the from_line options + if(enabled === false){ + return this.__resetField(); + } + let field = this.state.field.toString(encoding); + if(rtrim === true && wasQuoting === false){ + field = field.trimRight(); + } + if(cast === true){ + const [err, f] = this.__cast(field); + if(err !== undefined) return err; + field = f; + } + this.state.record.push(field); + // Increment record length if record size must not exceed a limit + if(max_record_size !== 0 && typeof field === 'string'){ + this.state.record_length += field.length; + } + this.__resetField(); + }, + __resetField: function(){ + this.state.field.reset(); + this.state.wasQuoting = false; + }, + __push: function(record, push){ + const {on_record} = this.options; + if(on_record !== undefined){ + const info = this.__infoRecord(); + try{ + record = on_record.call(null, record, info); + }catch(err){ + return err; + } + if(record === undefined || record === null){ return; } + } + push(record); + }, + // Return a tuple with the error and the casted value + __cast: function(field){ + const {columns, relax_column_count} = this.options; + const isColumns = Array.isArray(columns); + // Dont loose time calling cast + // because the final record is an object + // and this field can't be associated to a key present in columns + if(isColumns === true && relax_column_count && this.options.columns.length <= this.state.record.length){ + return [undefined, undefined]; + } + if(this.state.castField !== null){ + try{ + const info = this.__infoField(); + return [undefined, this.state.castField.call(null, field, info)]; + }catch(err){ + return [err]; + } + } + if(this.__isFloat(field)){ + return [undefined, parseFloat(field)]; + }else if(this.options.cast_date !== false){ const info = this.__infoField(); - return [undefined, this.state.castField.call(null, field, info)]; - }catch(err){ - return [err]; + return [undefined, this.options.cast_date.call(null, field, info)]; + } + return [undefined, field]; + }, + // Helper to test if a character is a space or a line delimiter + __isCharTrimable: function(chr){ + return chr === space || chr === tab || chr === cr || chr === nl || chr === np; + }, + // Keep it in case we implement the `cast_int` option + // __isInt(value){ + // // return Number.isInteger(parseInt(value)) + // // return !isNaN( parseInt( obj ) ); + // return /^(\-|\+)?[1-9][0-9]*$/.test(value) + // } + __isFloat: function(value){ + return (value - parseFloat(value) + 1) >= 0; // Borrowed from jquery + }, + __compareBytes: function(sourceBuf, targetBuf, targetPos, firstByte){ + if(sourceBuf[0] !== firstByte) return 0; + const sourceLength = sourceBuf.length; + for(let i = 1; i < sourceLength; i++){ + if(sourceBuf[i] !== targetBuf[targetPos+i]) return 0; + } + return sourceLength; + }, + __isDelimiter: function(buf, pos, chr){ + const {delimiter, ignore_last_delimiters} = this.options; + if(ignore_last_delimiters === true && this.state.record.length === this.options.columns.length - 1){ + return 0; + }else if(ignore_last_delimiters !== false && typeof ignore_last_delimiters === 'number' && this.state.record.length === ignore_last_delimiters - 1){ + return 0; + } + loop1: for(let i = 0; i < delimiter.length; i++){ + const del = delimiter[i]; + if(del[0] === chr){ + for(let j = 1; j < del.length; j++){ + if(del[j] !== buf[pos+j]) continue loop1; + } + return del.length; + } } - } - if(this.__isFloat(field)){ - return [undefined, parseFloat(field)]; - }else if(this.options.cast_date !== false){ - const info = this.__infoField(); - return [undefined, this.options.cast_date.call(null, field, info)]; - } - return [undefined, field]; - } - // Helper to test if a character is a space or a line delimiter - __isCharTrimable(chr){ - return chr === space || chr === tab || chr === cr || chr === nl || chr === np; - } - // Keep it in case we implement the `cast_int` option - // __isInt(value){ - // // return Number.isInteger(parseInt(value)) - // // return !isNaN( parseInt( obj ) ); - // return /^(\-|\+)?[1-9][0-9]*$/.test(value) - // } - __isFloat(value){ - return (value - parseFloat(value) + 1) >= 0; // Borrowed from jquery - } - __compareBytes(sourceBuf, targetBuf, targetPos, firstByte){ - if(sourceBuf[0] !== firstByte) return 0; - const sourceLength = sourceBuf.length; - for(let i = 1; i < sourceLength; i++){ - if(sourceBuf[i] !== targetBuf[targetPos+i]) return 0; - } - return sourceLength; - } - __needMoreData(i, bufLen, end){ - if(end) return false; - const {quote} = this.options; - const {quoting, needMoreDataSize, recordDelimiterMaxLength} = this.state; - const numOfCharLeft = bufLen - i - 1; - const requiredLength = Math.max( - needMoreDataSize, - // Skip if the remaining buffer smaller than record delimiter - recordDelimiterMaxLength, - // Skip if the remaining buffer can be record delimiter following the closing quote - // 1 is for quote.length - quoting ? (quote.length + recordDelimiterMaxLength) : 0, - ); - return numOfCharLeft < requiredLength; - } - __isDelimiter(buf, pos, chr){ - const {delimiter, ignore_last_delimiters} = this.options; - if(ignore_last_delimiters === true && this.state.record.length === this.options.columns.length - 1){ return 0; - }else if(ignore_last_delimiters !== false && typeof ignore_last_delimiters === 'number' && this.state.record.length === ignore_last_delimiters - 1){ - return 0; - } - loop1: for(let i = 0; i < delimiter.length; i++){ - const del = delimiter[i]; - if(del[0] === chr){ - for(let j = 1; j < del.length; j++){ - if(del[j] !== buf[pos+j]) continue loop1; + }, + __isRecordDelimiter: function(chr, buf, pos){ + const {record_delimiter} = this.options; + const recordDelimiterLength = record_delimiter.length; + loop1: for(let i = 0; i < recordDelimiterLength; i++){ + const rd = record_delimiter[i]; + const rdLength = rd.length; + if(rd[0] !== chr){ + continue; } - return del.length; - } - } - return 0; - } - __isRecordDelimiter(chr, buf, pos){ - const {record_delimiter} = this.options; - const recordDelimiterLength = record_delimiter.length; - loop1: for(let i = 0; i < recordDelimiterLength; i++){ - const rd = record_delimiter[i]; - const rdLength = rd.length; - if(rd[0] !== chr){ - continue; - } - for(let j = 1; j < rdLength; j++){ - if(rd[j] !== buf[pos+j]){ - continue loop1; + for(let j = 1; j < rdLength; j++){ + if(rd[j] !== buf[pos+j]){ + continue loop1; + } } + return rd.length; } - return rd.length; - } - return 0; - } - __isEscape(buf, pos, chr){ - const {escape} = this.options; - if(escape === null) return false; - const l = escape.length; - if(escape[0] === chr){ + return 0; + }, + __isEscape: function(buf, pos, chr){ + const {escape} = this.options; + if(escape === null) return false; + const l = escape.length; + if(escape[0] === chr){ + for(let i = 0; i < l; i++){ + if(escape[i] !== buf[pos+i]){ + return false; + } + } + return true; + } + return false; + }, + __isQuote: function(buf, pos){ + const {quote} = this.options; + if(quote === null) return false; + const l = quote.length; for(let i = 0; i < l; i++){ - if(escape[i] !== buf[pos+i]){ + if(quote[i] !== buf[pos+i]){ return false; } } return true; - } - return false; - } - __isQuote(buf, pos){ - const {quote} = this.options; - if(quote === null) return false; - const l = quote.length; - for(let i = 0; i < l; i++){ - if(quote[i] !== buf[pos+i]){ - return false; - } - } - return true; - } - __autoDiscoverRecordDelimiter(buf, pos){ - const {encoding} = this.options; - const chr = buf[pos]; - if(chr === cr){ - if(buf[pos+1] === nl){ - this.options.record_delimiter.push(Buffer.from('\r\n', encoding)); - this.state.recordDelimiterMaxLength = 2; - return 2; - }else { - this.options.record_delimiter.push(Buffer.from('\r', encoding)); + }, + __autoDiscoverRecordDelimiter: function(buf, pos){ + const {encoding} = this.options; + const chr = buf[pos]; + if(chr === cr){ + if(buf[pos+1] === nl){ + this.options.record_delimiter.push(Buffer.from('\r\n', encoding)); + this.state.recordDelimiterMaxLength = 2; + return 2; + }else { + this.options.record_delimiter.push(Buffer.from('\r', encoding)); + this.state.recordDelimiterMaxLength = 1; + return 1; + } + }else if(chr === nl){ + this.options.record_delimiter.push(Buffer.from('\n', encoding)); this.state.recordDelimiterMaxLength = 1; return 1; } - }else if(chr === nl){ - this.options.record_delimiter.push(Buffer.from('\n', encoding)); - this.state.recordDelimiterMaxLength = 1; - return 1; - } - return 0; - } - __error(msg){ - const {encoding, raw, skip_records_with_error} = this.options; - const err = typeof msg === 'string' ? new Error(msg) : msg; - if(skip_records_with_error){ - this.state.recordHasError = true; - this.emit('skip', err, raw ? this.state.rawBuffer.toString(encoding) : undefined); - return undefined; - }else { - return err; + return 0; + }, + __error: function(msg){ + const {encoding, raw, skip_records_with_error} = this.options; + const err = typeof msg === 'string' ? new Error(msg) : msg; + if(skip_records_with_error){ + this.state.recordHasError = true; + if(this.options.on_skip !== undefined){ + this.options.on_skip(err, raw ? this.state.rawBuffer.toString(encoding) : undefined); + } + // this.emit('skip', err, raw ? this.state.rawBuffer.toString(encoding) : undefined); + return undefined; + }else { + return err; + } + }, + __infoDataSet: function(){ + return { + ...this.info, + columns: this.options.columns + }; + }, + __infoRecord: function(){ + const {columns, raw, encoding} = this.options; + return { + ...this.__infoDataSet(), + error: this.state.error, + header: columns === true, + index: this.state.record.length, + raw: raw ? this.state.rawBuffer.toString(encoding) : undefined + }; + }, + __infoField: function(){ + const {columns} = this.options; + const isColumns = Array.isArray(columns); + return { + ...this.__infoRecord(), + column: isColumns === true ? + (columns.length > this.state.record.length ? + columns[this.state.record.length].name : + null + ) : + this.state.record.length, + quoting: this.state.wasQuoting, + }; } - } - __infoDataSet(){ - return { - ...this.info, - columns: this.options.columns + }; +}; + +class Parser extends stream.Transform { + constructor(opts = {}){ + super({...{readableObjectMode: true}, ...opts, encoding: null}); + this.api = transform$1(opts); + this.api.options.on_skip = (err, chunk) => { + this.emit('skip', err, chunk); }; + // Backward compatibility + this.state = this.api.state; + this.options = this.api.options; + this.info = this.api.info; } - __infoRecord(){ - const {columns, raw, encoding} = this.options; - return { - ...this.__infoDataSet(), - error: this.state.error, - header: columns === true, - index: this.state.record.length, - raw: raw ? this.state.rawBuffer.toString(encoding) : undefined - }; + // Implementation of `Transform._transform` + _transform(buf, encoding, callback){ + if(this.state.stop === true){ + return; + } + const err = this.api.parse(buf, false, (record) => { + this.push.call(this, record); + }, () => { + this.push.call(this, null); + }); + if(err !== undefined){ + this.state.stop = true; + } + callback(err); } - __infoField(){ - const {columns} = this.options; - const isColumns = Array.isArray(columns); - return { - ...this.__infoRecord(), - column: isColumns === true ? - (columns.length > this.state.record.length ? - columns[this.state.record.length].name : - null - ) : - this.state.record.length, - quoting: this.state.wasQuoting, - }; + // Implementation of `Transform._flush` + _flush(callback){ + if(this.state.stop === true){ + return; + } + const err = this.api.parse(undefined, true, (record) => { + this.push.call(this, record); + }, () => { + this.push.call(this, null); + }); + callback(err); } } @@ -1505,7 +1569,7 @@ const parse = function(){ const type = typeof argument; if(data === undefined && (typeof argument === 'string' || Buffer.isBuffer(argument))){ data = argument; - }else if(options === undefined && isObject$1(argument)){ + }else if(options === undefined && is_object$1(argument)){ options = argument; }else if(callback === undefined && type === 'function'){ callback = argument; @@ -1530,10 +1594,10 @@ const parse = function(){ } }); parser.on('error', function(err){ - callback(err, undefined, parser.__infoDataSet()); + callback(err, undefined, parser.api.__infoDataSet()); }); parser.on('end', function(){ - callback(undefined, records, parser.__infoDataSet()); + callback(undefined, records, parser.api.__infoDataSet()); }); } if(data !== undefined){ @@ -1696,8 +1760,6 @@ const transform = function(){ return transformer; }; -const bom_utf8 = Buffer.from([239, 187, 191]); - class CsvError extends Error { constructor(code, message, ...contexts) { if(Array.isArray(message)) message = message.join(' '); @@ -1715,16 +1777,10 @@ class CsvError extends Error { } } -const isObject = function(obj){ +const is_object = function(obj){ return typeof obj === 'object' && obj !== null && ! Array.isArray(obj); }; -const underscore = function(str){ - return str.replace(/([A-Z])/g, function(_, match){ - return '_' + match.toLowerCase(); - }); -}; - // Lodash implementation of `get` const charCodeOfDot = '.'.charCodeAt(0); @@ -1802,36 +1858,435 @@ const get = function(object, path){ return (index && index === length) ? object : undefined; }; +const normalize_columns = function(columns){ + if(columns === undefined || columns === null){ + return [undefined, undefined]; + } + if(typeof columns !== 'object'){ + return [Error('Invalid option "columns": expect an array or an object')]; + } + if(!Array.isArray(columns)){ + const newcolumns = []; + for(const k in columns){ + newcolumns.push({ + key: k, + header: columns[k] + }); + } + columns = newcolumns; + }else { + const newcolumns = []; + for(const column of columns){ + if(typeof column === 'string'){ + newcolumns.push({ + key: column, + header: column + }); + }else if(typeof column === 'object' && column !== null && !Array.isArray(column)){ + if(!column.key){ + return [Error('Invalid column definition: property "key" is required')]; + } + if(column.header === undefined){ + column.header = column.key; + } + newcolumns.push(column); + }else { + return [Error('Invalid column definition: expect a string or an object')]; + } + } + columns = newcolumns; + } + return [undefined, columns]; +}; + +const underscore = function(str){ + return str.replace(/([A-Z])/g, function(_, match){ + return '_' + match.toLowerCase(); + }); +}; + +const normalize_options = function(opts) { + const options = {}; + // Merge with user options + for(const opt in opts){ + options[underscore(opt)] = opts[opt]; + } + // Normalize option `bom` + if(options.bom === undefined || options.bom === null || options.bom === false){ + options.bom = false; + }else if(options.bom !== true){ + return [new CsvError('CSV_OPTION_BOOLEAN_INVALID_TYPE', [ + 'option `bom` is optional and must be a boolean value,', + `got ${JSON.stringify(options.bom)}` + ])]; + } + // Normalize option `delimiter` + if(options.delimiter === undefined || options.delimiter === null){ + options.delimiter = ','; + }else if(Buffer.isBuffer(options.delimiter)){ + options.delimiter = options.delimiter.toString(); + }else if(typeof options.delimiter !== 'string'){ + return [new CsvError('CSV_OPTION_DELIMITER_INVALID_TYPE', [ + 'option `delimiter` must be a buffer or a string,', + `got ${JSON.stringify(options.delimiter)}` + ])]; + } + // Normalize option `quote` + if(options.quote === undefined || options.quote === null){ + options.quote = '"'; + }else if(options.quote === true){ + options.quote = '"'; + }else if(options.quote === false){ + options.quote = ''; + }else if (Buffer.isBuffer(options.quote)){ + options.quote = options.quote.toString(); + }else if(typeof options.quote !== 'string'){ + return [new CsvError('CSV_OPTION_QUOTE_INVALID_TYPE', [ + 'option `quote` must be a boolean, a buffer or a string,', + `got ${JSON.stringify(options.quote)}` + ])]; + } + // Normalize option `quoted` + if(options.quoted === undefined || options.quoted === null){ + options.quoted = false; + } + // Normalize option `quoted_empty` + if(options.quoted_empty === undefined || options.quoted_empty === null){ + options.quoted_empty = undefined; + } + // Normalize option `quoted_match` + if(options.quoted_match === undefined || options.quoted_match === null || options.quoted_match === false){ + options.quoted_match = null; + }else if(!Array.isArray(options.quoted_match)){ + options.quoted_match = [options.quoted_match]; + } + if(options.quoted_match){ + for(const quoted_match of options.quoted_match){ + const isString = typeof quoted_match === 'string'; + const isRegExp = quoted_match instanceof RegExp; + if(!isString && !isRegExp){ + return [Error(`Invalid Option: quoted_match must be a string or a regex, got ${JSON.stringify(quoted_match)}`)]; + } + } + } + // Normalize option `quoted_string` + if(options.quoted_string === undefined || options.quoted_string === null){ + options.quoted_string = false; + } + // Normalize option `eof` + if(options.eof === undefined || options.eof === null){ + options.eof = true; + } + // Normalize option `escape` + if(options.escape === undefined || options.escape === null){ + options.escape = '"'; + }else if(Buffer.isBuffer(options.escape)){ + options.escape = options.escape.toString(); + }else if(typeof options.escape !== 'string'){ + return [Error(`Invalid Option: escape must be a buffer or a string, got ${JSON.stringify(options.escape)}`)]; + } + if (options.escape.length > 1){ + return [Error(`Invalid Option: escape must be one character, got ${options.escape.length} characters`)]; + } + // Normalize option `header` + if(options.header === undefined || options.header === null){ + options.header = false; + } + // Normalize option `columns` + const [errColumns, columns] = normalize_columns(options.columns); + if(errColumns !== undefined) return [errColumns]; + options.columns = columns; + // Normalize option `quoted` + if(options.quoted === undefined || options.quoted === null){ + options.quoted = false; + } + // Normalize option `cast` + if(options.cast === undefined || options.cast === null){ + options.cast = {}; + } + // Normalize option cast.bigint + if(options.cast.bigint === undefined || options.cast.bigint === null){ + // Cast boolean to string by default + options.cast.bigint = value => '' + value; + } + // Normalize option cast.boolean + if(options.cast.boolean === undefined || options.cast.boolean === null){ + // Cast boolean to string by default + options.cast.boolean = value => value ? '1' : ''; + } + // Normalize option cast.date + if(options.cast.date === undefined || options.cast.date === null){ + // Cast date to timestamp string by default + options.cast.date = value => '' + value.getTime(); + } + // Normalize option cast.number + if(options.cast.number === undefined || options.cast.number === null){ + // Cast number to string using native casting by default + options.cast.number = value => '' + value; + } + // Normalize option cast.object + if(options.cast.object === undefined || options.cast.object === null){ + // Stringify object as JSON by default + options.cast.object = value => JSON.stringify(value); + } + // Normalize option cast.string + if(options.cast.string === undefined || options.cast.string === null){ + // Leave string untouched + options.cast.string = function(value){return value;}; + } + // Normalize option `on_record` + if(options.on_record !== undefined && typeof options.on_record !== 'function'){ + return [Error(`Invalid Option: "on_record" must be a function.`)]; + } + // Normalize option `record_delimiter` + if(options.record_delimiter === undefined || options.record_delimiter === null){ + options.record_delimiter = '\n'; + }else if(Buffer.isBuffer(options.record_delimiter)){ + options.record_delimiter = options.record_delimiter.toString(); + }else if(typeof options.record_delimiter !== 'string'){ + return [Error(`Invalid Option: record_delimiter must be a buffer or a string, got ${JSON.stringify(options.record_delimiter)}`)]; + } + switch(options.record_delimiter){ + case 'auto': + options.record_delimiter = null; + break; + case 'unix': + options.record_delimiter = "\n"; + break; + case 'mac': + options.record_delimiter = "\r"; + break; + case 'windows': + options.record_delimiter = "\r\n"; + break; + case 'ascii': + options.record_delimiter = "\u001e"; + break; + case 'unicode': + options.record_delimiter = "\u2028"; + break; + } + return [undefined, options]; +}; + +const bom_utf8 = Buffer.from([239, 187, 191]); + +const stringifier = function(options, state, info){ + return { + options: options, + state: state, + info: info, + __transform: function(chunk, push){ + // Chunk validation + if(!Array.isArray(chunk) && typeof chunk !== 'object'){ + return Error(`Invalid Record: expect an array or an object, got ${JSON.stringify(chunk)}`); + } + // Detect columns from the first record + if(this.info.records === 0){ + if(Array.isArray(chunk)){ + if(this.options.header === true && this.options.columns === undefined){ + return Error('Undiscoverable Columns: header option requires column option or object records'); + } + }else if(this.options.columns === undefined){ + const [err, columns] = normalize_columns(Object.keys(chunk)); + if(err) return; + this.options.columns = columns; + } + } + // Emit the header + if(this.info.records === 0){ + this.bom(push); + const err = this.headers(push); + if(err) return err; + } + // Emit and stringify the record if an object or an array + try{ + // this.emit('record', chunk, this.info.records); + if(this.options.on_record){ + this.options.on_record(chunk, this.info.records); + } + }catch(err){ + return err; + } + // Convert the record into a string + let err, chunk_string; + if(this.options.eof){ + [err, chunk_string] = this.stringify(chunk); + if(err) return err; + if(chunk_string === undefined){ + return; + }else { + chunk_string = chunk_string + this.options.record_delimiter; + } + }else { + [err, chunk_string] = this.stringify(chunk); + if(err) return err; + if(chunk_string === undefined){ + return; + }else { + if(this.options.header || this.info.records){ + chunk_string = this.options.record_delimiter + chunk_string; + } + } + } + // Emit the csv + this.info.records++; + push(chunk_string); + }, + stringify: function(chunk, chunkIsHeader=false){ + if(typeof chunk !== 'object'){ + return [undefined, chunk]; + } + const {columns} = this.options; + const record = []; + // Record is an array + if(Array.isArray(chunk)){ + // We are getting an array but the user has specified output columns. In + // this case, we respect the columns indexes + if(columns){ + chunk.splice(columns.length); + } + // Cast record elements + for(let i=0; i= 0; + const containsQuote = (quote !== '') && value.indexOf(quote) >= 0; + const containsEscape = value.indexOf(escape) >= 0 && (escape !== quote); + const containsRecordDelimiter = value.indexOf(record_delimiter) >= 0; + const quotedString = quoted_string && typeof field === 'string'; + let quotedMatch = quoted_match && quoted_match.filter(quoted_match => { + if(typeof quoted_match === 'string'){ + return value.indexOf(quoted_match) !== -1; + }else { + return quoted_match.test(value); + } + }); + quotedMatch = quotedMatch && quotedMatch.length > 0; + const shouldQuote = containsQuote === true || containsdelimiter || containsRecordDelimiter || quoted || quotedString || quotedMatch; + if(shouldQuote === true && containsEscape === true){ + const regexp = escape === '\\' + ? new RegExp(escape + escape, 'g') + : new RegExp(escape, 'g'); + value = value.replace(regexp, escape + escape); + } + if(containsQuote === true){ + const regexp = new RegExp(quote,'g'); + value = value.replace(regexp, escape + quote); + } + if(shouldQuote === true){ + value = quote + value + quote; + } + csvrecord += value; + }else if(quoted_empty === true || (field === '' && quoted_string === true && quoted_empty !== false)){ + csvrecord += quote + quote; + } + if(i !== record.length - 1){ + csvrecord += delimiter; + } + } + return [undefined, csvrecord]; + }, + bom: function(push){ + if(this.options.bom !== true){ + return; + } + push(bom_utf8); + }, + headers: function(push){ + if(this.options.header === false){ + return; + } + if(this.options.columns === undefined){ + return; + } + let err; + let headers = this.options.columns.map(column => column.header); + if(this.options.eof){ + [err, headers] = this.stringify(headers, true); + headers += this.options.record_delimiter; + }else { + [err, headers] = this.stringify(headers); + } + if(err) return err; + push(headers); + }, + __cast: function(value, context){ + const type = typeof value; + try{ + if(type === 'string'){ // Fine for 99% of the cases + return [undefined, this.options.cast.string(value, context)]; + }else if(type === 'bigint'){ + return [undefined, this.options.cast.bigint(value, context)]; + }else if(type === 'number'){ + return [undefined, this.options.cast.number(value, context)]; + }else if(type === 'boolean'){ + return [undefined, this.options.cast.boolean(value, context)]; + }else if(value instanceof Date){ + return [undefined, this.options.cast.date(value, context)]; + }else if(type === 'object' && value !== null){ + return [undefined, this.options.cast.object(value, context)]; + }else { + return [undefined, value, value]; + } + }catch(err){ + return [err]; + } + } + }; +}; + class Stringifier extends stream.Transform { constructor(opts = {}){ super({...{writableObjectMode: true}, ...opts}); - const options = {}; - let err; - // Merge with user options - for(const opt in opts){ - options[underscore(opt)] = opts[opt]; - } - if((err = this.normalize(options)) !== undefined) throw err; - switch(options.record_delimiter){ - case 'auto': - options.record_delimiter = null; - break; - case 'unix': - options.record_delimiter = "\n"; - break; - case 'mac': - options.record_delimiter = "\r"; - break; - case 'windows': - options.record_delimiter = "\r\n"; - break; - case 'ascii': - options.record_delimiter = "\u001e"; - break; - case 'unicode': - options.record_delimiter = "\u2028"; - break; - } + const [err, options] = normalize_options(opts); + if(err !== undefined) throw err; // Expose options this.options = options; // Internal state @@ -1842,145 +2297,16 @@ class Stringifier extends stream.Transform { this.info = { records: 0 }; - } - normalize(options){ - // Normalize option `bom` - if(options.bom === undefined || options.bom === null || options.bom === false){ - options.bom = false; - }else if(options.bom !== true){ - return new CsvError('CSV_OPTION_BOOLEAN_INVALID_TYPE', [ - 'option `bom` is optional and must be a boolean value,', - `got ${JSON.stringify(options.bom)}` - ]); - } - // Normalize option `delimiter` - if(options.delimiter === undefined || options.delimiter === null){ - options.delimiter = ','; - }else if(Buffer.isBuffer(options.delimiter)){ - options.delimiter = options.delimiter.toString(); - }else if(typeof options.delimiter !== 'string'){ - return new CsvError('CSV_OPTION_DELIMITER_INVALID_TYPE', [ - 'option `delimiter` must be a buffer or a string,', - `got ${JSON.stringify(options.delimiter)}` - ]); - } - // Normalize option `quote` - if(options.quote === undefined || options.quote === null){ - options.quote = '"'; - }else if(options.quote === true){ - options.quote = '"'; - }else if(options.quote === false){ - options.quote = ''; - }else if (Buffer.isBuffer(options.quote)){ - options.quote = options.quote.toString(); - }else if(typeof options.quote !== 'string'){ - return new CsvError('CSV_OPTION_QUOTE_INVALID_TYPE', [ - 'option `quote` must be a boolean, a buffer or a string,', - `got ${JSON.stringify(options.quote)}` - ]); - } - // Normalize option `quoted` - if(options.quoted === undefined || options.quoted === null){ - options.quoted = false; - } - // Normalize option `quoted_empty` - if(options.quoted_empty === undefined || options.quoted_empty === null){ - options.quoted_empty = undefined; - } - // Normalize option `quoted_match` - if(options.quoted_match === undefined || options.quoted_match === null || options.quoted_match === false){ - options.quoted_match = null; - }else if(!Array.isArray(options.quoted_match)){ - options.quoted_match = [options.quoted_match]; - } - if(options.quoted_match){ - for(const quoted_match of options.quoted_match){ - const isString = typeof quoted_match === 'string'; - const isRegExp = quoted_match instanceof RegExp; - if(!isString && !isRegExp){ - return Error(`Invalid Option: quoted_match must be a string or a regex, got ${JSON.stringify(quoted_match)}`); - } - } - } - // Normalize option `quoted_string` - if(options.quoted_string === undefined || options.quoted_string === null){ - options.quoted_string = false; - } - // Normalize option `eof` - if(options.eof === undefined || options.eof === null){ - options.eof = true; - } - // Normalize option `escape` - if(options.escape === undefined || options.escape === null){ - options.escape = '"'; - }else if(Buffer.isBuffer(options.escape)){ - options.escape = options.escape.toString(); - }else if(typeof options.escape !== 'string'){ - return Error(`Invalid Option: escape must be a buffer or a string, got ${JSON.stringify(options.escape)}`); - } - if (options.escape.length > 1){ - return Error(`Invalid Option: escape must be one character, got ${options.escape.length} characters`); - } - // Normalize option `header` - if(options.header === undefined || options.header === null){ - options.header = false; - } - // Normalize option `columns` - const [err, columns] = this.normalize_columns(options.columns); - if(err) return err; - options.columns = columns; - // Normalize option `quoted` - if(options.quoted === undefined || options.quoted === null){ - options.quoted = false; - } - // Normalize option `cast` - if(options.cast === undefined || options.cast === null){ - options.cast = {}; - } - // Normalize option cast.bigint - if(options.cast.bigint === undefined || options.cast.bigint === null){ - // Cast boolean to string by default - options.cast.bigint = value => '' + value; - } - // Normalize option cast.boolean - if(options.cast.boolean === undefined || options.cast.boolean === null){ - // Cast boolean to string by default - options.cast.boolean = value => value ? '1' : ''; - } - // Normalize option cast.date - if(options.cast.date === undefined || options.cast.date === null){ - // Cast date to timestamp string by default - options.cast.date = value => '' + value.getTime(); - } - // Normalize option cast.number - if(options.cast.number === undefined || options.cast.number === null){ - // Cast number to string using native casting by default - options.cast.number = value => '' + value; - } - // Normalize option cast.object - if(options.cast.object === undefined || options.cast.object === null){ - // Stringify object as JSON by default - options.cast.object = value => JSON.stringify(value); - } - // Normalize option cast.string - if(options.cast.string === undefined || options.cast.string === null){ - // Leave string untouched - options.cast.string = function(value){return value;}; - } - // Normalize option `record_delimiter` - if(options.record_delimiter === undefined || options.record_delimiter === null){ - options.record_delimiter = '\n'; - }else if(Buffer.isBuffer(options.record_delimiter)){ - options.record_delimiter = options.record_delimiter.toString(); - }else if(typeof options.record_delimiter !== 'string'){ - return Error(`Invalid Option: record_delimiter must be a buffer or a string, got ${JSON.stringify(options.record_delimiter)}`); - } + this.api = stringifier(this.options, this.state, this.info); + this.api.options.on_record = (...args) => { + this.emit('record', ...args); + }; } _transform(chunk, encoding, callback){ if(this.state.stop === true){ return; } - const err = this.__transform(chunk); + const err = this.api.__transform(chunk, this.push.bind(this)); if(err !== undefined){ this.state.stop = true; } @@ -1993,251 +2319,12 @@ class Stringifier extends stream.Transform { return; } if(this.info.records === 0){ - this.bom(); - const err = this.headers(); + this.api.bom(this.push.bind(this)); + const err = this.api.headers(this.push.bind(this)); if(err) callback(err); } callback(); } - __transform(chunk){ - // Chunk validation - if(!Array.isArray(chunk) && typeof chunk !== 'object'){ - return Error(`Invalid Record: expect an array or an object, got ${JSON.stringify(chunk)}`); - } - // Detect columns from the first record - if(this.info.records === 0){ - if(Array.isArray(chunk)){ - if(this.options.header === true && this.options.columns === undefined){ - return Error('Undiscoverable Columns: header option requires column option or object records'); - } - }else if(this.options.columns === undefined){ - const [err, columns] = this.normalize_columns(Object.keys(chunk)); - if(err) return; - this.options.columns = columns; - } - } - // Emit the header - if(this.info.records === 0){ - this.bom(); - const err = this.headers(); - if(err) return err; - } - // Emit and stringify the record if an object or an array - try{ - this.emit('record', chunk, this.info.records); - }catch(err){ - return err; - } - // Convert the record into a string - let err, chunk_string; - if(this.options.eof){ - [err, chunk_string] = this.stringify(chunk); - if(err) return err; - if(chunk_string === undefined){ - return; - }else { - chunk_string = chunk_string + this.options.record_delimiter; - } - }else { - [err, chunk_string] = this.stringify(chunk); - if(err) return err; - if(chunk_string === undefined){ - return; - }else { - if(this.options.header || this.info.records){ - chunk_string = this.options.record_delimiter + chunk_string; - } - } - } - // Emit the csv - this.info.records++; - this.push(chunk_string); - } - stringify(chunk, chunkIsHeader=false){ - if(typeof chunk !== 'object'){ - return [undefined, chunk]; - } - const {columns} = this.options; - const record = []; - // Record is an array - if(Array.isArray(chunk)){ - // We are getting an array but the user has specified output columns. In - // this case, we respect the columns indexes - if(columns){ - chunk.splice(columns.length); - } - // Cast record elements - for(let i=0; i= 0; - const containsQuote = (quote !== '') && value.indexOf(quote) >= 0; - const containsEscape = value.indexOf(escape) >= 0 && (escape !== quote); - const containsRecordDelimiter = value.indexOf(record_delimiter) >= 0; - const quotedString = quoted_string && typeof field === 'string'; - let quotedMatch = quoted_match && quoted_match.filter(quoted_match => { - if(typeof quoted_match === 'string'){ - return value.indexOf(quoted_match) !== -1; - }else { - return quoted_match.test(value); - } - }); - quotedMatch = quotedMatch && quotedMatch.length > 0; - const shouldQuote = containsQuote === true || containsdelimiter || containsRecordDelimiter || quoted || quotedString || quotedMatch; - if(shouldQuote === true && containsEscape === true){ - const regexp = escape === '\\' - ? new RegExp(escape + escape, 'g') - : new RegExp(escape, 'g'); - value = value.replace(regexp, escape + escape); - } - if(containsQuote === true){ - const regexp = new RegExp(quote,'g'); - value = value.replace(regexp, escape + quote); - } - if(shouldQuote === true){ - value = quote + value + quote; - } - csvrecord += value; - }else if(quoted_empty === true || (field === '' && quoted_string === true && quoted_empty !== false)){ - csvrecord += quote + quote; - } - if(i !== record.length - 1){ - csvrecord += delimiter; - } - } - return [undefined, csvrecord]; - } - bom(){ - if(this.options.bom !== true){ - return; - } - this.push(bom_utf8); - } - headers(){ - if(this.options.header === false){ - return; - } - if(this.options.columns === undefined){ - return; - } - let err; - let headers = this.options.columns.map(column => column.header); - if(this.options.eof){ - [err, headers] = this.stringify(headers, true); - headers += this.options.record_delimiter; - }else { - [err, headers] = this.stringify(headers); - } - if(err) return err; - this.push(headers); - } - __cast(value, context){ - const type = typeof value; - try{ - if(type === 'string'){ // Fine for 99% of the cases - return [undefined, this.options.cast.string(value, context)]; - }else if(type === 'bigint'){ - return [undefined, this.options.cast.bigint(value, context)]; - }else if(type === 'number'){ - return [undefined, this.options.cast.number(value, context)]; - }else if(type === 'boolean'){ - return [undefined, this.options.cast.boolean(value, context)]; - }else if(value instanceof Date){ - return [undefined, this.options.cast.date(value, context)]; - }else if(type === 'object' && value !== null){ - return [undefined, this.options.cast.object(value, context)]; - }else { - return [undefined, value, value]; - } - }catch(err){ - return [err]; - } - } - normalize_columns(columns){ - if(columns === undefined || columns === null){ - return []; - } - if(typeof columns !== 'object'){ - return [Error('Invalid option "columns": expect an array or an object')]; - } - if(!Array.isArray(columns)){ - const newcolumns = []; - for(const k in columns){ - newcolumns.push({ - key: k, - header: columns[k] - }); - } - columns = newcolumns; - }else { - const newcolumns = []; - for(const column of columns){ - if(typeof column === 'string'){ - newcolumns.push({ - key: column, - header: column - }); - }else if(typeof column === 'object' && column !== undefined && !Array.isArray(column)){ - if(!column.key){ - return [Error('Invalid column definition: property "key" is required')]; - } - if(column.header === undefined){ - column.header = column.key; - } - newcolumns.push(column); - }else { - return [Error('Invalid column definition: expect a string or an object')]; - } - } - columns = newcolumns; - } - return [undefined, columns]; - } } const stringify = function(){ @@ -2247,7 +2334,7 @@ const stringify = function(){ const type = typeof argument; if(data === undefined && (Array.isArray(argument))){ data = argument; - }else if(options === undefined && isObject(argument)){ + }else if(options === undefined && is_object(argument)){ options = argument; }else if(callback === undefined && type === 'function'){ callback = argument; diff --git a/packages/csv/dist/cjs/sync.cjs b/packages/csv/dist/cjs/sync.cjs index 60de4bfbd..62dc2e86f 100644 --- a/packages/csv/dist/cjs/sync.cjs +++ b/packages/csv/dist/cjs/sync.cjs @@ -10,20 +10,64 @@ function _interopDefaultLegacy (e) { return e && typeof e === 'object' && 'defau var stream__default = /*#__PURE__*/_interopDefaultLegacy(stream); var util__default = /*#__PURE__*/_interopDefaultLegacy(util); -const Generator = function(options = {}){ +const init_state$1 = (options) => { + // State + return { + start_time: options.duration ? Date.now() : null, + fixed_size_buffer: '', + count_written: 0, + count_created: 0, + }; +}; + +// Generate a random number between 0 and 1 with 2 decimals. The function is idempotent if it detect the "seed" option. +const random = function(options={}){ + if(options.seed){ + return options.seed = options.seed * Math.PI * 100 % 100 / 100; + }else { + return Math.random(); + } +}; + +const types = { + // Generate an ASCII value. + ascii: function({options}){ + const column = []; + const nb_chars = Math.ceil(random(options) * options.maxWordLength); + for(let i=0; i { // Convert Stream Readable options if underscored - if(options.high_water_mark){ - options.highWaterMark = options.high_water_mark; + if(opts.high_water_mark){ + opts.highWaterMark = opts.high_water_mark; } - if(options.object_mode){ - options.objectMode = options.object_mode; + if(opts.object_mode){ + opts.objectMode = opts.object_mode; } - // Call parent constructor - stream__default["default"].Readable.call(this, options); // Clone and camelize options - this.options = {}; - for(const k in options){ - this.options[Generator.camelize(k)] = options[k]; + const options = {}; + for(const k in opts){ + options[camelize(k)] = opts[k]; } // Normalize options const dft = { @@ -41,110 +85,100 @@ const Generator = function(options = {}){ sleep: 0, }; for(const k in dft){ - if(this.options[k] === undefined){ - this.options[k] = dft[k]; + if(options[k] === undefined){ + options[k] = dft[k]; } } // Default values - if(this.options.eof === true){ - this.options.eof = this.options.rowDelimiter; + if(options.eof === true){ + options.eof = options.rowDelimiter; } - // State - this._ = { - start_time: this.options.duration ? Date.now() : null, - fixed_size_buffer: '', - count_written: 0, - count_created: 0, - }; - if(typeof this.options.columns === 'number'){ - this.options.columns = new Array(this.options.columns); + if(typeof options.columns === 'number'){ + options.columns = new Array(options.columns); } - const accepted_header_types = Object.keys(Generator).filter((t) => (!['super_', 'camelize'].includes(t))); - for(let i = 0; i < this.options.columns.length; i++){ - const v = this.options.columns[i] || 'ascii'; + const accepted_header_types = Object.keys(types).filter((t) => (!['super_', 'camelize'].includes(t))); + for(let i = 0; i < options.columns.length; i++){ + const v = options.columns[i] || 'ascii'; if(typeof v === 'string'){ if(!accepted_header_types.includes(v)){ throw Error(`Invalid column type: got "${v}", default values are ${JSON.stringify(accepted_header_types)}`); } - this.options.columns[i] = Generator[v]; + options.columns[i] = types[v]; } } - return this; + return options; }; -util__default["default"].inherits(Generator, stream__default["default"].Readable); -// Generate a random number between 0 and 1 with 2 decimals. The function is idempotent if it detect the "seed" option. -Generator.prototype.random = function(){ - if(this.options.seed){ - return this.options.seed = this.options.seed * Math.PI * 100 % 100 / 100; - }else { - return Math.random(); - } -}; -// Stop the generation. -Generator.prototype.end = function(){ - this.push(null); -}; -// Put new data into the read queue. -Generator.prototype._read = function(size){ +const read = (options, state, size, push, close) => { // Already started const data = []; - let length = this._.fixed_size_buffer.length; + let length = state.fixed_size_buffer.length; if(length !== 0){ - data.push(this._.fixed_size_buffer); + data.push(state.fixed_size_buffer); } // eslint-disable-next-line while(true){ // Time for some rest: flush first and stop later - if((this._.count_created === this.options.length) || (this.options.end && Date.now() > this.options.end) || (this.options.duration && Date.now() > this._.start_time + this.options.duration)){ + if((state.count_created === options.length) || (options.end && Date.now() > options.end) || (options.duration && Date.now() > state.start_time + options.duration)){ // Flush if(data.length){ - if(this.options.objectMode){ + if(options.objectMode){ for(const record of data){ - this.__push(record); + push(record); } }else { - this.__push(data.join('') + (this.options.eof ? this.options.eof : '')); + push(data.join('') + (options.eof ? options.eof : '')); } - this._.end = true; + state.end = true; }else { - this.push(null); + close(); } return; } // Create the record let record = []; let recordLength; - this.options.columns.forEach((fn) => { - record.push(fn(this)); + options.columns.forEach((fn) => { + const result = fn({options: options, state: state}); + const type = typeof result; + if(result !== null && type !== 'string' && type !== 'number'){ + throw Error([ + 'INVALID_VALUE:', + 'values returned by column function must be', + 'a string, a number or null,', + `got ${JSON.stringify(result)}` + ].join(' ')); + } + record.push(result); }); // Obtain record length - if(this.options.objectMode){ + if(options.objectMode){ recordLength = 0; // recordLength is currently equal to the number of columns // This is wrong and shall equal to 1 record only - for(const column of record) + for(const column of record){ recordLength += column.length; + } }else { // Stringify the record - record = (this._.count_created === 0 ? '' : this.options.rowDelimiter)+record.join(this.options.delimiter); + record = (state.count_created === 0 ? '' : options.rowDelimiter)+record.join(options.delimiter); recordLength = record.length; } - this._.count_created++; + state.count_created++; if(length + recordLength > size){ - if(this.options.objectMode){ + if(options.objectMode){ data.push(record); for(const record of data){ - this.__push(record); + push(record); } }else { - if(this.options.fixedSize){ - this._.fixed_size_buffer = record.substr(size - length); + if(options.fixedSize){ + state.fixed_size_buffer = record.substr(size - length); data.push(record.substr(0, size - length)); }else { data.push(record); } - this.__push(data.join('')); + push(data.join('')); } return; } @@ -152,42 +186,41 @@ Generator.prototype._read = function(size){ data.push(record); } }; + +const Generator = function(options = {}){ + this.options = normalize_options$2(options); + // Call parent constructor + stream__default["default"].Readable.call(this, this.options); + this.state = init_state$1(this.options); + return this; +}; +util__default["default"].inherits(Generator, stream__default["default"].Readable); + +// Stop the generation. +Generator.prototype.end = function(){ + this.push(null); +}; +// Put new data into the read queue. +Generator.prototype._read = function(size){ + const self = this; + read(this.options, this.state, size, function(chunk) { + self.__push(chunk); + }, function(){ + self.push(null); + }); +}; // Put new data into the read queue. Generator.prototype.__push = function(record){ + // console.log('push', record) const push = () => { - this._.count_written++; + this.state.count_written++; this.push(record); - if(this._.end === true){ + if(this.state.end === true){ return this.push(null); } }; this.options.sleep > 0 ? setTimeout(push, this.options.sleep) : push(); }; -// Generate an ASCII value. -Generator.ascii = function(gen){ - // Column - const column = []; - const nb_chars = Math.ceil(gen.random() * gen.options.maxWordLength); - for(let i=0; i delimiter.length), + // Skip if the remaining buffer can be escape sequence + options.quote !== null ? options.quote.length : 0, + ), + previousBuf: undefined, + quoting: false, + stop: false, + rawBuffer: new ResizeableBuffer(100), + record: [], + recordHasError: false, + record_length: 0, + recordDelimiterMaxLength: options.record_delimiter.length === 0 ? 2 : Math.max(...options.record_delimiter.map((v) => v.length)), + trimChars: [Buffer.from(' ', options.encoding)[0], Buffer.from('\t', options.encoding)[0]], + wasQuoting: false, + wasRowDelimiter: false + }; }; -class CsvError$1 extends Error { - constructor(code, message, options, ...contexts) { - if(Array.isArray(message)) message = message.join(' '); - super(message); - if(Error.captureStackTrace !== undefined){ - Error.captureStackTrace(this, CsvError$1); - } - this.code = code; - for(const context of contexts){ - for(const key in context){ - const value = context[key]; - this[key] = Buffer.isBuffer(value) ? value.toString(options.encoding) : value == null ? value : JSON.parse(JSON.stringify(value)); - } - } - } -} - const underscore$1 = function(str){ return str.replace(/([A-Z])/g, function(_, match){ return '_' + match.toLowerCase(); }); }; -const isObject$1 = function(obj){ - return (typeof obj === 'object' && obj !== null && !Array.isArray(obj)); -}; - -const isRecordEmpty = function(record){ - return record.every((field) => field == null || field.toString && field.toString().trim() === ''); -}; - -const normalizeColumnsArray = function(columns){ - const normalizedColumns = []; - for(let i = 0, l = columns.length; i < l; i++){ - const column = columns[i]; - if(column === undefined || column === null || column === false){ - normalizedColumns[i] = { disabled: true }; - }else if(typeof column === 'string'){ - normalizedColumns[i] = { name: column }; - }else if(isObject$1(column)){ - if(typeof column.name !== 'string'){ - throw new CsvError$1('CSV_OPTION_COLUMNS_MISSING_NAME', [ - 'Option columns missing name:', - `property "name" is required at position ${i}`, - 'when column is an object literal' - ]); - } - normalizedColumns[i] = column; - }else { - throw new CsvError$1('CSV_INVALID_COLUMN_DEFINITION', [ - 'Invalid column definition:', - 'expect a string or a literal object,', - `got ${JSON.stringify(column)} at position ${i}` - ]); - } - } - return normalizedColumns; -}; - -class Parser extends stream.Transform { - constructor(opts = {}){ - super({...{readableObjectMode: true}, ...opts, encoding: null}); - this.__originalOptions = opts; - this.__normalizeOptions(opts); - } - __normalizeOptions(opts){ - const options = {}; - // Merge with user options - for(const opt in opts){ - options[underscore$1(opt)] = opts[opt]; - } - // Normalize option `encoding` - // Note: defined first because other options depends on it - // to convert chars/strings into buffers. - if(options.encoding === undefined || options.encoding === true){ - options.encoding = 'utf8'; - }else if(options.encoding === null || options.encoding === false){ - options.encoding = null; - }else if(typeof options.encoding !== 'string' && options.encoding !== null){ - throw new CsvError$1('CSV_INVALID_OPTION_ENCODING', [ - 'Invalid option encoding:', - 'encoding must be a string or null to return a buffer,', - `got ${JSON.stringify(options.encoding)}` - ], options); - } - // Normalize option `bom` - if(options.bom === undefined || options.bom === null || options.bom === false){ - options.bom = false; - }else if(options.bom !== true){ - throw new CsvError$1('CSV_INVALID_OPTION_BOM', [ - 'Invalid option bom:', 'bom must be true,', - `got ${JSON.stringify(options.bom)}` - ], options); - } - // Normalize option `cast` - let fnCastField = null; - if(options.cast === undefined || options.cast === null || options.cast === false || options.cast === ''){ - options.cast = undefined; - }else if(typeof options.cast === 'function'){ - fnCastField = options.cast; - options.cast = true; - }else if(options.cast !== true){ - throw new CsvError$1('CSV_INVALID_OPTION_CAST', [ - 'Invalid option cast:', 'cast must be true or a function,', - `got ${JSON.stringify(options.cast)}` - ], options); - } - // Normalize option `cast_date` - if(options.cast_date === undefined || options.cast_date === null || options.cast_date === false || options.cast_date === ''){ - options.cast_date = false; - }else if(options.cast_date === true){ - options.cast_date = function(value){ - const date = Date.parse(value); - return !isNaN(date) ? new Date(date) : value; - }; - }else { - throw new CsvError$1('CSV_INVALID_OPTION_CAST_DATE', [ - 'Invalid option cast_date:', 'cast_date must be true or a function,', - `got ${JSON.stringify(options.cast_date)}` - ], options); - } - // Normalize option `columns` - let fnFirstLineToHeaders = null; - if(options.columns === true){ - // Fields in the first line are converted as-is to columns - fnFirstLineToHeaders = undefined; - }else if(typeof options.columns === 'function'){ - fnFirstLineToHeaders = options.columns; - options.columns = true; - }else if(Array.isArray(options.columns)){ - options.columns = normalizeColumnsArray(options.columns); - }else if(options.columns === undefined || options.columns === null || options.columns === false){ - options.columns = false; - }else { - throw new CsvError$1('CSV_INVALID_OPTION_COLUMNS', [ - 'Invalid option columns:', - 'expect an array, a function or true,', - `got ${JSON.stringify(options.columns)}` +const normalize_options$1 = function(opts){ + const options = {}; + // Merge with user options + for(const opt in opts){ + options[underscore$1(opt)] = opts[opt]; + } + // Normalize option `encoding` + // Note: defined first because other options depends on it + // to convert chars/strings into buffers. + if(options.encoding === undefined || options.encoding === true){ + options.encoding = 'utf8'; + }else if(options.encoding === null || options.encoding === false){ + options.encoding = null; + }else if(typeof options.encoding !== 'string' && options.encoding !== null){ + throw new CsvError$1('CSV_INVALID_OPTION_ENCODING', [ + 'Invalid option encoding:', + 'encoding must be a string or null to return a buffer,', + `got ${JSON.stringify(options.encoding)}` + ], options); + } + // Normalize option `bom` + if(options.bom === undefined || options.bom === null || options.bom === false){ + options.bom = false; + }else if(options.bom !== true){ + throw new CsvError$1('CSV_INVALID_OPTION_BOM', [ + 'Invalid option bom:', 'bom must be true,', + `got ${JSON.stringify(options.bom)}` + ], options); + } + // Normalize option `cast` + options.cast_function = null; + if(options.cast === undefined || options.cast === null || options.cast === false || options.cast === ''){ + options.cast = undefined; + }else if(typeof options.cast === 'function'){ + options.cast_function = options.cast; + options.cast = true; + }else if(options.cast !== true){ + throw new CsvError$1('CSV_INVALID_OPTION_CAST', [ + 'Invalid option cast:', 'cast must be true or a function,', + `got ${JSON.stringify(options.cast)}` + ], options); + } + // Normalize option `cast_date` + if(options.cast_date === undefined || options.cast_date === null || options.cast_date === false || options.cast_date === ''){ + options.cast_date = false; + }else if(options.cast_date === true){ + options.cast_date = function(value){ + const date = Date.parse(value); + return !isNaN(date) ? new Date(date) : value; + }; + }else { + throw new CsvError$1('CSV_INVALID_OPTION_CAST_DATE', [ + 'Invalid option cast_date:', 'cast_date must be true or a function,', + `got ${JSON.stringify(options.cast_date)}` + ], options); + } + // Normalize option `columns` + options.cast_first_line_to_header = null; + if(options.columns === true){ + // Fields in the first line are converted as-is to columns + options.cast_first_line_to_header = undefined; + }else if(typeof options.columns === 'function'){ + options.cast_first_line_to_header = options.columns; + options.columns = true; + }else if(Array.isArray(options.columns)){ + options.columns = normalize_columns_array(options.columns); + }else if(options.columns === undefined || options.columns === null || options.columns === false){ + options.columns = false; + }else { + throw new CsvError$1('CSV_INVALID_OPTION_COLUMNS', [ + 'Invalid option columns:', + 'expect an array, a function or true,', + `got ${JSON.stringify(options.columns)}` + ], options); + } + // Normalize option `group_columns_by_name` + if(options.group_columns_by_name === undefined || options.group_columns_by_name === null || options.group_columns_by_name === false){ + options.group_columns_by_name = false; + }else if(options.group_columns_by_name !== true){ + throw new CsvError$1('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ + 'Invalid option group_columns_by_name:', + 'expect an boolean,', + `got ${JSON.stringify(options.group_columns_by_name)}` + ], options); + }else if(options.columns === false){ + throw new CsvError$1('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ + 'Invalid option group_columns_by_name:', + 'the `columns` mode must be activated.' + ], options); + } + // Normalize option `comment` + if(options.comment === undefined || options.comment === null || options.comment === false || options.comment === ''){ + options.comment = null; + }else { + if(typeof options.comment === 'string'){ + options.comment = Buffer.from(options.comment, options.encoding); + } + if(!Buffer.isBuffer(options.comment)){ + throw new CsvError$1('CSV_INVALID_OPTION_COMMENT', [ + 'Invalid option comment:', + 'comment must be a buffer or a string,', + `got ${JSON.stringify(options.comment)}` ], options); } - // Normalize option `group_columns_by_name` - if(options.group_columns_by_name === undefined || options.group_columns_by_name === null || options.group_columns_by_name === false){ - options.group_columns_by_name = false; - }else if(options.group_columns_by_name !== true){ - throw new CsvError$1('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ - 'Invalid option group_columns_by_name:', - 'expect an boolean,', - `got ${JSON.stringify(options.group_columns_by_name)}` - ], options); - }else if(options.columns === false){ - throw new CsvError$1('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ - 'Invalid option group_columns_by_name:', - 'the `columns` mode must be activated.' - ], options); + } + // Normalize option `delimiter` + const delimiter_json = JSON.stringify(options.delimiter); + if(!Array.isArray(options.delimiter)) options.delimiter = [options.delimiter]; + if(options.delimiter.length === 0){ + throw new CsvError$1('CSV_INVALID_OPTION_DELIMITER', [ + 'Invalid option delimiter:', + 'delimiter must be a non empty string or buffer or array of string|buffer,', + `got ${delimiter_json}` + ], options); + } + options.delimiter = options.delimiter.map(function(delimiter){ + if(delimiter === undefined || delimiter === null || delimiter === false){ + return Buffer.from(',', options.encoding); } - // Normalize option `comment` - if(options.comment === undefined || options.comment === null || options.comment === false || options.comment === ''){ - options.comment = null; - }else { - if(typeof options.comment === 'string'){ - options.comment = Buffer.from(options.comment, options.encoding); - } - if(!Buffer.isBuffer(options.comment)){ - throw new CsvError$1('CSV_INVALID_OPTION_COMMENT', [ - 'Invalid option comment:', - 'comment must be a buffer or a string,', - `got ${JSON.stringify(options.comment)}` - ], options); - } + if(typeof delimiter === 'string'){ + delimiter = Buffer.from(delimiter, options.encoding); } - // Normalize option `delimiter` - const delimiter_json = JSON.stringify(options.delimiter); - if(!Array.isArray(options.delimiter)) options.delimiter = [options.delimiter]; - if(options.delimiter.length === 0){ + if(!Buffer.isBuffer(delimiter) || delimiter.length === 0){ throw new CsvError$1('CSV_INVALID_OPTION_DELIMITER', [ 'Invalid option delimiter:', 'delimiter must be a non empty string or buffer or array of string|buffer,', `got ${delimiter_json}` ], options); } - options.delimiter = options.delimiter.map(function(delimiter){ - if(delimiter === undefined || delimiter === null || delimiter === false){ - return Buffer.from(',', options.encoding); - } - if(typeof delimiter === 'string'){ - delimiter = Buffer.from(delimiter, options.encoding); - } - if(!Buffer.isBuffer(delimiter) || delimiter.length === 0){ - throw new CsvError$1('CSV_INVALID_OPTION_DELIMITER', [ - 'Invalid option delimiter:', - 'delimiter must be a non empty string or buffer or array of string|buffer,', - `got ${delimiter_json}` - ], options); - } - return delimiter; - }); - // Normalize option `escape` - if(options.escape === undefined || options.escape === true){ - options.escape = Buffer.from('"', options.encoding); - }else if(typeof options.escape === 'string'){ - options.escape = Buffer.from(options.escape, options.encoding); - }else if (options.escape === null || options.escape === false){ - options.escape = null; + return delimiter; + }); + // Normalize option `escape` + if(options.escape === undefined || options.escape === true){ + options.escape = Buffer.from('"', options.encoding); + }else if(typeof options.escape === 'string'){ + options.escape = Buffer.from(options.escape, options.encoding); + }else if (options.escape === null || options.escape === false){ + options.escape = null; + } + if(options.escape !== null){ + if(!Buffer.isBuffer(options.escape)){ + throw new Error(`Invalid Option: escape must be a buffer, a string or a boolean, got ${JSON.stringify(options.escape)}`); + } + } + // Normalize option `from` + if(options.from === undefined || options.from === null){ + options.from = 1; + }else { + if(typeof options.from === 'string' && /\d+/.test(options.from)){ + options.from = parseInt(options.from); } - if(options.escape !== null){ - if(!Buffer.isBuffer(options.escape)){ - throw new Error(`Invalid Option: escape must be a buffer, a string or a boolean, got ${JSON.stringify(options.escape)}`); + if(Number.isInteger(options.from)){ + if(options.from < 0){ + throw new Error(`Invalid Option: from must be a positive integer, got ${JSON.stringify(opts.from)}`); } - } - // Normalize option `from` - if(options.from === undefined || options.from === null){ - options.from = 1; }else { - if(typeof options.from === 'string' && /\d+/.test(options.from)){ - options.from = parseInt(options.from); - } - if(Number.isInteger(options.from)){ - if(options.from < 0){ - throw new Error(`Invalid Option: from must be a positive integer, got ${JSON.stringify(opts.from)}`); - } - }else { - throw new Error(`Invalid Option: from must be an integer, got ${JSON.stringify(options.from)}`); - } + throw new Error(`Invalid Option: from must be an integer, got ${JSON.stringify(options.from)}`); } - // Normalize option `from_line` - if(options.from_line === undefined || options.from_line === null){ - options.from_line = 1; - }else { - if(typeof options.from_line === 'string' && /\d+/.test(options.from_line)){ - options.from_line = parseInt(options.from_line); - } - if(Number.isInteger(options.from_line)){ - if(options.from_line <= 0){ - throw new Error(`Invalid Option: from_line must be a positive integer greater than 0, got ${JSON.stringify(opts.from_line)}`); - } - }else { - throw new Error(`Invalid Option: from_line must be an integer, got ${JSON.stringify(opts.from_line)}`); - } + } + // Normalize option `from_line` + if(options.from_line === undefined || options.from_line === null){ + options.from_line = 1; + }else { + if(typeof options.from_line === 'string' && /\d+/.test(options.from_line)){ + options.from_line = parseInt(options.from_line); } - // Normalize options `ignore_last_delimiters` - if(options.ignore_last_delimiters === undefined || options.ignore_last_delimiters === null){ - options.ignore_last_delimiters = false; - }else if(typeof options.ignore_last_delimiters === 'number'){ - options.ignore_last_delimiters = Math.floor(options.ignore_last_delimiters); - if(options.ignore_last_delimiters === 0){ - options.ignore_last_delimiters = false; + if(Number.isInteger(options.from_line)){ + if(options.from_line <= 0){ + throw new Error(`Invalid Option: from_line must be a positive integer greater than 0, got ${JSON.stringify(opts.from_line)}`); } - }else if(typeof options.ignore_last_delimiters !== 'boolean'){ - throw new CsvError$1('CSV_INVALID_OPTION_IGNORE_LAST_DELIMITERS', [ - 'Invalid option `ignore_last_delimiters`:', - 'the value must be a boolean value or an integer,', - `got ${JSON.stringify(options.ignore_last_delimiters)}` - ], options); - } - if(options.ignore_last_delimiters === true && options.columns === false){ - throw new CsvError$1('CSV_IGNORE_LAST_DELIMITERS_REQUIRES_COLUMNS', [ - 'The option `ignore_last_delimiters`', - 'requires the activation of the `columns` option' - ], options); - } - // Normalize option `info` - if(options.info === undefined || options.info === null || options.info === false){ - options.info = false; - }else if(options.info !== true){ - throw new Error(`Invalid Option: info must be true, got ${JSON.stringify(options.info)}`); - } - // Normalize option `max_record_size` - if(options.max_record_size === undefined || options.max_record_size === null || options.max_record_size === false){ - options.max_record_size = 0; - }else if(Number.isInteger(options.max_record_size) && options.max_record_size >= 0);else if(typeof options.max_record_size === 'string' && /\d+/.test(options.max_record_size)){ - options.max_record_size = parseInt(options.max_record_size); }else { - throw new Error(`Invalid Option: max_record_size must be a positive integer, got ${JSON.stringify(options.max_record_size)}`); - } - // Normalize option `objname` - if(options.objname === undefined || options.objname === null || options.objname === false){ - options.objname = undefined; - }else if(Buffer.isBuffer(options.objname)){ - if(options.objname.length === 0){ - throw new Error(`Invalid Option: objname must be a non empty buffer`); - } - if(options.encoding === null);else { - options.objname = options.objname.toString(options.encoding); - } - }else if(typeof options.objname === 'string'){ - if(options.objname.length === 0){ - throw new Error(`Invalid Option: objname must be a non empty string`); - } - // Great, nothing to do - }else if(typeof options.objname === 'number');else { - throw new Error(`Invalid Option: objname must be a string or a buffer, got ${options.objname}`); + throw new Error(`Invalid Option: from_line must be an integer, got ${JSON.stringify(opts.from_line)}`); } - if(options.objname !== undefined){ - if(typeof options.objname === 'number'){ - if(options.columns !== false){ - throw Error('Invalid Option: objname index cannot be combined with columns or be defined as a field'); - } - }else { // A string or a buffer - if(options.columns === false){ - throw Error('Invalid Option: objname field must be combined with columns or be defined as an index'); - } - } + } + // Normalize options `ignore_last_delimiters` + if(options.ignore_last_delimiters === undefined || options.ignore_last_delimiters === null){ + options.ignore_last_delimiters = false; + }else if(typeof options.ignore_last_delimiters === 'number'){ + options.ignore_last_delimiters = Math.floor(options.ignore_last_delimiters); + if(options.ignore_last_delimiters === 0){ + options.ignore_last_delimiters = false; } - // Normalize option `on_record` - if(options.on_record === undefined || options.on_record === null){ - options.on_record = undefined; - }else if(typeof options.on_record !== 'function'){ - throw new CsvError$1('CSV_INVALID_OPTION_ON_RECORD', [ - 'Invalid option `on_record`:', - 'expect a function,', - `got ${JSON.stringify(options.on_record)}` + }else if(typeof options.ignore_last_delimiters !== 'boolean'){ + throw new CsvError$1('CSV_INVALID_OPTION_IGNORE_LAST_DELIMITERS', [ + 'Invalid option `ignore_last_delimiters`:', + 'the value must be a boolean value or an integer,', + `got ${JSON.stringify(options.ignore_last_delimiters)}` + ], options); + } + if(options.ignore_last_delimiters === true && options.columns === false){ + throw new CsvError$1('CSV_IGNORE_LAST_DELIMITERS_REQUIRES_COLUMNS', [ + 'The option `ignore_last_delimiters`', + 'requires the activation of the `columns` option' + ], options); + } + // Normalize option `info` + if(options.info === undefined || options.info === null || options.info === false){ + options.info = false; + }else if(options.info !== true){ + throw new Error(`Invalid Option: info must be true, got ${JSON.stringify(options.info)}`); + } + // Normalize option `max_record_size` + if(options.max_record_size === undefined || options.max_record_size === null || options.max_record_size === false){ + options.max_record_size = 0; + }else if(Number.isInteger(options.max_record_size) && options.max_record_size >= 0);else if(typeof options.max_record_size === 'string' && /\d+/.test(options.max_record_size)){ + options.max_record_size = parseInt(options.max_record_size); + }else { + throw new Error(`Invalid Option: max_record_size must be a positive integer, got ${JSON.stringify(options.max_record_size)}`); + } + // Normalize option `objname` + if(options.objname === undefined || options.objname === null || options.objname === false){ + options.objname = undefined; + }else if(Buffer.isBuffer(options.objname)){ + if(options.objname.length === 0){ + throw new Error(`Invalid Option: objname must be a non empty buffer`); + } + if(options.encoding === null);else { + options.objname = options.objname.toString(options.encoding); + } + }else if(typeof options.objname === 'string'){ + if(options.objname.length === 0){ + throw new Error(`Invalid Option: objname must be a non empty string`); + } + // Great, nothing to do + }else if(typeof options.objname === 'number');else { + throw new Error(`Invalid Option: objname must be a string or a buffer, got ${options.objname}`); + } + if(options.objname !== undefined){ + if(typeof options.objname === 'number'){ + if(options.columns !== false){ + throw Error('Invalid Option: objname index cannot be combined with columns or be defined as a field'); + } + }else { // A string or a buffer + if(options.columns === false){ + throw Error('Invalid Option: objname field must be combined with columns or be defined as an index'); + } + } + } + // Normalize option `on_record` + if(options.on_record === undefined || options.on_record === null){ + options.on_record = undefined; + }else if(typeof options.on_record !== 'function'){ + throw new CsvError$1('CSV_INVALID_OPTION_ON_RECORD', [ + 'Invalid option `on_record`:', + 'expect a function,', + `got ${JSON.stringify(options.on_record)}` + ], options); + } + // Normalize option `quote` + if(options.quote === null || options.quote === false || options.quote === ''){ + options.quote = null; + }else { + if(options.quote === undefined || options.quote === true){ + options.quote = Buffer.from('"', options.encoding); + }else if(typeof options.quote === 'string'){ + options.quote = Buffer.from(options.quote, options.encoding); + } + if(!Buffer.isBuffer(options.quote)){ + throw new Error(`Invalid Option: quote must be a buffer or a string, got ${JSON.stringify(options.quote)}`); + } + } + // Normalize option `raw` + if(options.raw === undefined || options.raw === null || options.raw === false){ + options.raw = false; + }else if(options.raw !== true){ + throw new Error(`Invalid Option: raw must be true, got ${JSON.stringify(options.raw)}`); + } + // Normalize option `record_delimiter` + if(options.record_delimiter === undefined){ + options.record_delimiter = []; + }else if(typeof options.record_delimiter === 'string' || Buffer.isBuffer(options.record_delimiter)){ + if(options.record_delimiter.length === 0){ + throw new CsvError$1('CSV_INVALID_OPTION_RECORD_DELIMITER', [ + 'Invalid option `record_delimiter`:', + 'value must be a non empty string or buffer,', + `got ${JSON.stringify(options.record_delimiter)}` ], options); } - // Normalize option `quote` - if(options.quote === null || options.quote === false || options.quote === ''){ - options.quote = null; - }else { - if(options.quote === undefined || options.quote === true){ - options.quote = Buffer.from('"', options.encoding); - }else if(typeof options.quote === 'string'){ - options.quote = Buffer.from(options.quote, options.encoding); - } - if(!Buffer.isBuffer(options.quote)){ - throw new Error(`Invalid Option: quote must be a buffer or a string, got ${JSON.stringify(options.quote)}`); - } - } - // Normalize option `raw` - if(options.raw === undefined || options.raw === null || options.raw === false){ - options.raw = false; - }else if(options.raw !== true){ - throw new Error(`Invalid Option: raw must be true, got ${JSON.stringify(options.raw)}`); - } - // Normalize option `record_delimiter` - if(options.record_delimiter === undefined){ - options.record_delimiter = []; - }else if(typeof options.record_delimiter === 'string' || Buffer.isBuffer(options.record_delimiter)){ - if(options.record_delimiter.length === 0){ - throw new CsvError$1('CSV_INVALID_OPTION_RECORD_DELIMITER', [ - 'Invalid option `record_delimiter`:', - 'value must be a non empty string or buffer,', - `got ${JSON.stringify(options.record_delimiter)}` - ], options); - } - options.record_delimiter = [options.record_delimiter]; - }else if(!Array.isArray(options.record_delimiter)){ + options.record_delimiter = [options.record_delimiter]; + }else if(!Array.isArray(options.record_delimiter)){ + throw new CsvError$1('CSV_INVALID_OPTION_RECORD_DELIMITER', [ + 'Invalid option `record_delimiter`:', + 'value must be a string, a buffer or array of string|buffer,', + `got ${JSON.stringify(options.record_delimiter)}` + ], options); + } + options.record_delimiter = options.record_delimiter.map(function(rd, i){ + if(typeof rd !== 'string' && ! Buffer.isBuffer(rd)){ throw new CsvError$1('CSV_INVALID_OPTION_RECORD_DELIMITER', [ 'Invalid option `record_delimiter`:', - 'value must be a string, a buffer or array of string|buffer,', - `got ${JSON.stringify(options.record_delimiter)}` + 'value must be a string, a buffer or array of string|buffer', + `at index ${i},`, + `got ${JSON.stringify(rd)}` + ], options); + }else if(rd.length === 0){ + throw new CsvError$1('CSV_INVALID_OPTION_RECORD_DELIMITER', [ + 'Invalid option `record_delimiter`:', + 'value must be a non empty string or buffer', + `at index ${i},`, + `got ${JSON.stringify(rd)}` ], options); } - options.record_delimiter = options.record_delimiter.map(function(rd, i){ - if(typeof rd !== 'string' && ! Buffer.isBuffer(rd)){ - throw new CsvError$1('CSV_INVALID_OPTION_RECORD_DELIMITER', [ - 'Invalid option `record_delimiter`:', - 'value must be a string, a buffer or array of string|buffer', - `at index ${i},`, - `got ${JSON.stringify(rd)}` - ], options); - }else if(rd.length === 0){ - throw new CsvError$1('CSV_INVALID_OPTION_RECORD_DELIMITER', [ - 'Invalid option `record_delimiter`:', - 'value must be a non empty string or buffer', - `at index ${i},`, - `got ${JSON.stringify(rd)}` - ], options); - } - if(typeof rd === 'string'){ - rd = Buffer.from(rd, options.encoding); - } - return rd; - }); - // Normalize option `relax_column_count` - if(typeof options.relax_column_count === 'boolean');else if(options.relax_column_count === undefined || options.relax_column_count === null){ - options.relax_column_count = false; - }else { - throw new Error(`Invalid Option: relax_column_count must be a boolean, got ${JSON.stringify(options.relax_column_count)}`); - } - if(typeof options.relax_column_count_less === 'boolean');else if(options.relax_column_count_less === undefined || options.relax_column_count_less === null){ - options.relax_column_count_less = false; - }else { - throw new Error(`Invalid Option: relax_column_count_less must be a boolean, got ${JSON.stringify(options.relax_column_count_less)}`); - } - if(typeof options.relax_column_count_more === 'boolean');else if(options.relax_column_count_more === undefined || options.relax_column_count_more === null){ - options.relax_column_count_more = false; - }else { - throw new Error(`Invalid Option: relax_column_count_more must be a boolean, got ${JSON.stringify(options.relax_column_count_more)}`); - } - // Normalize option `relax_quotes` - if(typeof options.relax_quotes === 'boolean');else if(options.relax_quotes === undefined || options.relax_quotes === null){ - options.relax_quotes = false; - }else { - throw new Error(`Invalid Option: relax_quotes must be a boolean, got ${JSON.stringify(options.relax_quotes)}`); - } - // Normalize option `skip_empty_lines` - if(typeof options.skip_empty_lines === 'boolean');else if(options.skip_empty_lines === undefined || options.skip_empty_lines === null){ - options.skip_empty_lines = false; - }else { - throw new Error(`Invalid Option: skip_empty_lines must be a boolean, got ${JSON.stringify(options.skip_empty_lines)}`); - } - // Normalize option `skip_records_with_empty_values` - if(typeof options.skip_records_with_empty_values === 'boolean');else if(options.skip_records_with_empty_values === undefined || options.skip_records_with_empty_values === null){ - options.skip_records_with_empty_values = false; - }else { - throw new Error(`Invalid Option: skip_records_with_empty_values must be a boolean, got ${JSON.stringify(options.skip_records_with_empty_values)}`); - } - // Normalize option `skip_records_with_error` - if(typeof options.skip_records_with_error === 'boolean');else if(options.skip_records_with_error === undefined || options.skip_records_with_error === null){ - options.skip_records_with_error = false; - }else { - throw new Error(`Invalid Option: skip_records_with_error must be a boolean, got ${JSON.stringify(options.skip_records_with_error)}`); - } - // Normalize option `rtrim` - if(options.rtrim === undefined || options.rtrim === null || options.rtrim === false){ - options.rtrim = false; - }else if(options.rtrim !== true){ - throw new Error(`Invalid Option: rtrim must be a boolean, got ${JSON.stringify(options.rtrim)}`); - } - // Normalize option `ltrim` - if(options.ltrim === undefined || options.ltrim === null || options.ltrim === false){ - options.ltrim = false; - }else if(options.ltrim !== true){ - throw new Error(`Invalid Option: ltrim must be a boolean, got ${JSON.stringify(options.ltrim)}`); + if(typeof rd === 'string'){ + rd = Buffer.from(rd, options.encoding); } - // Normalize option `trim` - if(options.trim === undefined || options.trim === null || options.trim === false){ - options.trim = false; - }else if(options.trim !== true){ - throw new Error(`Invalid Option: trim must be a boolean, got ${JSON.stringify(options.trim)}`); - } - // Normalize options `trim`, `ltrim` and `rtrim` - if(options.trim === true && opts.ltrim !== false){ - options.ltrim = true; - }else if(options.ltrim !== true){ - options.ltrim = false; - } - if(options.trim === true && opts.rtrim !== false){ - options.rtrim = true; - }else if(options.rtrim !== true){ - options.rtrim = false; + return rd; + }); + // Normalize option `relax_column_count` + if(typeof options.relax_column_count === 'boolean');else if(options.relax_column_count === undefined || options.relax_column_count === null){ + options.relax_column_count = false; + }else { + throw new Error(`Invalid Option: relax_column_count must be a boolean, got ${JSON.stringify(options.relax_column_count)}`); + } + if(typeof options.relax_column_count_less === 'boolean');else if(options.relax_column_count_less === undefined || options.relax_column_count_less === null){ + options.relax_column_count_less = false; + }else { + throw new Error(`Invalid Option: relax_column_count_less must be a boolean, got ${JSON.stringify(options.relax_column_count_less)}`); + } + if(typeof options.relax_column_count_more === 'boolean');else if(options.relax_column_count_more === undefined || options.relax_column_count_more === null){ + options.relax_column_count_more = false; + }else { + throw new Error(`Invalid Option: relax_column_count_more must be a boolean, got ${JSON.stringify(options.relax_column_count_more)}`); + } + // Normalize option `relax_quotes` + if(typeof options.relax_quotes === 'boolean');else if(options.relax_quotes === undefined || options.relax_quotes === null){ + options.relax_quotes = false; + }else { + throw new Error(`Invalid Option: relax_quotes must be a boolean, got ${JSON.stringify(options.relax_quotes)}`); + } + // Normalize option `skip_empty_lines` + if(typeof options.skip_empty_lines === 'boolean');else if(options.skip_empty_lines === undefined || options.skip_empty_lines === null){ + options.skip_empty_lines = false; + }else { + throw new Error(`Invalid Option: skip_empty_lines must be a boolean, got ${JSON.stringify(options.skip_empty_lines)}`); + } + // Normalize option `skip_records_with_empty_values` + if(typeof options.skip_records_with_empty_values === 'boolean');else if(options.skip_records_with_empty_values === undefined || options.skip_records_with_empty_values === null){ + options.skip_records_with_empty_values = false; + }else { + throw new Error(`Invalid Option: skip_records_with_empty_values must be a boolean, got ${JSON.stringify(options.skip_records_with_empty_values)}`); + } + // Normalize option `skip_records_with_error` + if(typeof options.skip_records_with_error === 'boolean');else if(options.skip_records_with_error === undefined || options.skip_records_with_error === null){ + options.skip_records_with_error = false; + }else { + throw new Error(`Invalid Option: skip_records_with_error must be a boolean, got ${JSON.stringify(options.skip_records_with_error)}`); + } + // Normalize option `rtrim` + if(options.rtrim === undefined || options.rtrim === null || options.rtrim === false){ + options.rtrim = false; + }else if(options.rtrim !== true){ + throw new Error(`Invalid Option: rtrim must be a boolean, got ${JSON.stringify(options.rtrim)}`); + } + // Normalize option `ltrim` + if(options.ltrim === undefined || options.ltrim === null || options.ltrim === false){ + options.ltrim = false; + }else if(options.ltrim !== true){ + throw new Error(`Invalid Option: ltrim must be a boolean, got ${JSON.stringify(options.ltrim)}`); + } + // Normalize option `trim` + if(options.trim === undefined || options.trim === null || options.trim === false){ + options.trim = false; + }else if(options.trim !== true){ + throw new Error(`Invalid Option: trim must be a boolean, got ${JSON.stringify(options.trim)}`); + } + // Normalize options `trim`, `ltrim` and `rtrim` + if(options.trim === true && opts.ltrim !== false){ + options.ltrim = true; + }else if(options.ltrim !== true){ + options.ltrim = false; + } + if(options.trim === true && opts.rtrim !== false){ + options.rtrim = true; + }else if(options.rtrim !== true){ + options.rtrim = false; + } + // Normalize option `to` + if(options.to === undefined || options.to === null){ + options.to = -1; + }else { + if(typeof options.to === 'string' && /\d+/.test(options.to)){ + options.to = parseInt(options.to); } - // Normalize option `to` - if(options.to === undefined || options.to === null){ - options.to = -1; - }else { - if(typeof options.to === 'string' && /\d+/.test(options.to)){ - options.to = parseInt(options.to); - } - if(Number.isInteger(options.to)){ - if(options.to <= 0){ - throw new Error(`Invalid Option: to must be a positive integer greater than 0, got ${JSON.stringify(opts.to)}`); - } - }else { - throw new Error(`Invalid Option: to must be an integer, got ${JSON.stringify(opts.to)}`); + if(Number.isInteger(options.to)){ + if(options.to <= 0){ + throw new Error(`Invalid Option: to must be a positive integer greater than 0, got ${JSON.stringify(opts.to)}`); } - } - // Normalize option `to_line` - if(options.to_line === undefined || options.to_line === null){ - options.to_line = -1; }else { - if(typeof options.to_line === 'string' && /\d+/.test(options.to_line)){ - options.to_line = parseInt(options.to_line); - } - if(Number.isInteger(options.to_line)){ - if(options.to_line <= 0){ - throw new Error(`Invalid Option: to_line must be a positive integer greater than 0, got ${JSON.stringify(opts.to_line)}`); - } - }else { - throw new Error(`Invalid Option: to_line must be an integer, got ${JSON.stringify(opts.to_line)}`); - } + throw new Error(`Invalid Option: to must be an integer, got ${JSON.stringify(opts.to)}`); } - this.info = { - bytes: 0, - comment_lines: 0, - empty_lines: 0, - invalid_field_length: 0, - lines: 1, - records: 0 - }; - this.options = options; - this.state = { - bomSkipped: false, - bufBytesStart: 0, - castField: fnCastField, - commenting: false, - // Current error encountered by a record - error: undefined, - enabled: options.from_line === 1, - escaping: false, - escapeIsQuote: Buffer.isBuffer(options.escape) && Buffer.isBuffer(options.quote) && Buffer.compare(options.escape, options.quote) === 0, - // columns can be `false`, `true`, `Array` - expectedRecordLength: Array.isArray(options.columns) ? options.columns.length : undefined, - field: new ResizeableBuffer(20), - firstLineToHeaders: fnFirstLineToHeaders, - needMoreDataSize: Math.max( - // Skip if the remaining buffer smaller than comment - options.comment !== null ? options.comment.length : 0, - // Skip if the remaining buffer can be delimiter - ...options.delimiter.map((delimiter) => delimiter.length), - // Skip if the remaining buffer can be escape sequence - options.quote !== null ? options.quote.length : 0, - ), - previousBuf: undefined, - quoting: false, - stop: false, - rawBuffer: new ResizeableBuffer(100), - record: [], - recordHasError: false, - record_length: 0, - recordDelimiterMaxLength: options.record_delimiter.length === 0 ? 2 : Math.max(...options.record_delimiter.map((v) => v.length)), - trimChars: [Buffer.from(' ', options.encoding)[0], Buffer.from('\t', options.encoding)[0]], - wasQuoting: false, - wasRowDelimiter: false - }; } - // Implementation of `Transform._transform` - _transform(buf, encoding, callback){ - if(this.state.stop === true){ - return; - } - const err = this.__parse(buf, false); - if(err !== undefined){ - this.state.stop = true; - } - callback(err); - } - // Implementation of `Transform._flush` - _flush(callback){ - if(this.state.stop === true){ - return; + // Normalize option `to_line` + if(options.to_line === undefined || options.to_line === null){ + options.to_line = -1; + }else { + if(typeof options.to_line === 'string' && /\d+/.test(options.to_line)){ + options.to_line = parseInt(options.to_line); } - const err = this.__parse(undefined, true); - callback(err); - } - // Central parser implementation - __parse(nextBuf, end){ - const {bom, comment, escape, from_line, ltrim, max_record_size, quote, raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options; - let {record_delimiter} = this.options; - const {bomSkipped, previousBuf, rawBuffer, escapeIsQuote} = this.state; - let buf; - if(previousBuf === undefined){ - if(nextBuf === undefined){ - // Handle empty string - this.push(null); - return; - }else { - buf = nextBuf; + if(Number.isInteger(options.to_line)){ + if(options.to_line <= 0){ + throw new Error(`Invalid Option: to_line must be a positive integer greater than 0, got ${JSON.stringify(opts.to_line)}`); } - }else if(previousBuf !== undefined && nextBuf === undefined){ - buf = previousBuf; }else { - buf = Buffer.concat([previousBuf, nextBuf]); + throw new Error(`Invalid Option: to_line must be an integer, got ${JSON.stringify(opts.to_line)}`); } - // Handle UTF BOM - if(bomSkipped === false){ - if(bom === false){ - this.state.bomSkipped = true; - }else if(buf.length < 3){ - // No enough data - if(end === false){ - // Wait for more data - this.state.previousBuf = buf; + } + return options; +}; + +const isRecordEmpty = function(record){ + return record.every((field) => field == null || field.toString && field.toString().trim() === ''); +}; + +// white space characters +// https://en.wikipedia.org/wiki/Whitespace_character +// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions/Character_Classes#Types +// \f\n\r\t\v\u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff +const tab = 9; +const nl = 10; // \n, 0x0A in hexadecimal, 10 in decimal +const np = 12; +const cr = 13; // \r, 0x0D in hexadécimal, 13 in decimal +const space = 32; +const boms = { + // Note, the following are equals: + // Buffer.from("\ufeff") + // Buffer.from([239, 187, 191]) + // Buffer.from('EFBBBF', 'hex') + 'utf8': Buffer.from([239, 187, 191]), + // Note, the following are equals: + // Buffer.from "\ufeff", 'utf16le + // Buffer.from([255, 254]) + 'utf16le': Buffer.from([255, 254]) +}; + +const transform$1 = function(original_options = {}) { + const info = { + bytes: 0, + comment_lines: 0, + empty_lines: 0, + invalid_field_length: 0, + lines: 1, + records: 0 + }; + const options = normalize_options$1(original_options); + return { + info: info, + original_options: original_options, + options: options, + state: init_state(options), + __needMoreData: function(i, bufLen, end){ + if(end) return false; + const {quote} = this.options; + const {quoting, needMoreDataSize, recordDelimiterMaxLength} = this.state; + const numOfCharLeft = bufLen - i - 1; + const requiredLength = Math.max( + needMoreDataSize, + // Skip if the remaining buffer smaller than record delimiter + recordDelimiterMaxLength, + // Skip if the remaining buffer can be record delimiter following the closing quote + // 1 is for quote.length + quoting ? (quote.length + recordDelimiterMaxLength) : 0, + ); + return numOfCharLeft < requiredLength; + }, + // Central parser implementation + parse: function(nextBuf, end, push, close){ + const {bom, comment, escape, from_line, ltrim, max_record_size, quote, raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options; + let {record_delimiter} = this.options; + const {bomSkipped, previousBuf, rawBuffer, escapeIsQuote} = this.state; + let buf; + if(previousBuf === undefined){ + if(nextBuf === undefined){ + // Handle empty string + close(); return; + }else { + buf = nextBuf; } + }else if(previousBuf !== undefined && nextBuf === undefined){ + buf = previousBuf; }else { - for(const encoding in boms){ - if(boms[encoding].compare(buf, 0, boms[encoding].length) === 0){ - // Skip BOM - const bomLength = boms[encoding].length; - this.state.bufBytesStart += bomLength; - buf = buf.slice(bomLength); - // Renormalize original options with the new encoding - this.__normalizeOptions({...this.__originalOptions, encoding: encoding}); - break; + buf = Buffer.concat([previousBuf, nextBuf]); + } + // Handle UTF BOM + if(bomSkipped === false){ + if(bom === false){ + this.state.bomSkipped = true; + }else if(buf.length < 3){ + // No enough data + if(end === false){ + // Wait for more data + this.state.previousBuf = buf; + return; } + }else { + for(const encoding in boms){ + if(boms[encoding].compare(buf, 0, boms[encoding].length) === 0){ + // Skip BOM + const bomLength = boms[encoding].length; + this.state.bufBytesStart += bomLength; + buf = buf.slice(bomLength); + // Renormalize original options with the new encoding + this.options = normalize_options$1({...this.original_options, encoding: encoding}); + break; + } + } + this.state.bomSkipped = true; } - this.state.bomSkipped = true; - } - } - const bufLen = buf.length; - let pos; - for(pos = 0; pos < bufLen; pos++){ - // Ensure we get enough space to look ahead - // There should be a way to move this out of the loop - if(this.__needMoreData(pos, bufLen, end)){ - break; - } - if(this.state.wasRowDelimiter === true){ - this.info.lines++; - this.state.wasRowDelimiter = false; - } - if(to_line !== -1 && this.info.lines > to_line){ - this.state.stop = true; - this.push(null); - return; } - // Auto discovery of record_delimiter, unix, mac and windows supported - if(this.state.quoting === false && record_delimiter.length === 0){ - const record_delimiterCount = this.__autoDiscoverRecordDelimiter(buf, pos); - if(record_delimiterCount){ - record_delimiter = this.options.record_delimiter; + const bufLen = buf.length; + let pos; + for(pos = 0; pos < bufLen; pos++){ + // Ensure we get enough space to look ahead + // There should be a way to move this out of the loop + if(this.__needMoreData(pos, bufLen, end)){ + break; } - } - const chr = buf[pos]; - if(raw === true){ - rawBuffer.append(chr); - } - if((chr === cr || chr === nl) && this.state.wasRowDelimiter === false){ - this.state.wasRowDelimiter = true; - } - // Previous char was a valid escape char - // treat the current char as a regular char - if(this.state.escaping === true){ - this.state.escaping = false; - }else { - // Escape is only active inside quoted fields - // We are quoting, the char is an escape chr and there is a chr to escape - // if(escape !== null && this.state.quoting === true && chr === escape && pos + 1 < bufLen){ - if(escape !== null && this.state.quoting === true && this.__isEscape(buf, pos, chr) && pos + escape.length < bufLen){ - if(escapeIsQuote){ - if(this.__isQuote(buf, pos+escape.length)){ - this.state.escaping = true; - pos += escape.length - 1; - continue; - } - }else { - this.state.escaping = true; - pos += escape.length - 1; - continue; + if(this.state.wasRowDelimiter === true){ + this.info.lines++; + this.state.wasRowDelimiter = false; + } + if(to_line !== -1 && this.info.lines > to_line){ + this.state.stop = true; + close(); + return; + } + // Auto discovery of record_delimiter, unix, mac and windows supported + if(this.state.quoting === false && record_delimiter.length === 0){ + const record_delimiterCount = this.__autoDiscoverRecordDelimiter(buf, pos); + if(record_delimiterCount){ + record_delimiter = this.options.record_delimiter; } } - // Not currently escaping and chr is a quote - // TODO: need to compare bytes instead of single char - if(this.state.commenting === false && this.__isQuote(buf, pos)){ - if(this.state.quoting === true){ - const nextChr = buf[pos+quote.length]; - const isNextChrTrimable = rtrim && this.__isCharTrimable(nextChr); - const isNextChrComment = comment !== null && this.__compareBytes(comment, buf, pos+quote.length, nextChr); - const isNextChrDelimiter = this.__isDelimiter(buf, pos+quote.length, nextChr); - const isNextChrRecordDelimiter = record_delimiter.length === 0 ? this.__autoDiscoverRecordDelimiter(buf, pos+quote.length) : this.__isRecordDelimiter(nextChr, buf, pos+quote.length); - // Escape a quote - // Treat next char as a regular character - if(escape !== null && this.__isEscape(buf, pos, chr) && this.__isQuote(buf, pos + escape.length)){ + const chr = buf[pos]; + if(raw === true){ + rawBuffer.append(chr); + } + if((chr === cr || chr === nl) && this.state.wasRowDelimiter === false){ + this.state.wasRowDelimiter = true; + } + // Previous char was a valid escape char + // treat the current char as a regular char + if(this.state.escaping === true){ + this.state.escaping = false; + }else { + // Escape is only active inside quoted fields + // We are quoting, the char is an escape chr and there is a chr to escape + // if(escape !== null && this.state.quoting === true && chr === escape && pos + 1 < bufLen){ + if(escape !== null && this.state.quoting === true && this.__isEscape(buf, pos, chr) && pos + escape.length < bufLen){ + if(escapeIsQuote){ + if(this.__isQuote(buf, pos+escape.length)){ + this.state.escaping = true; + pos += escape.length - 1; + continue; + } + }else { + this.state.escaping = true; pos += escape.length - 1; - }else if(!nextChr || isNextChrDelimiter || isNextChrRecordDelimiter || isNextChrComment || isNextChrTrimable){ - this.state.quoting = false; - this.state.wasQuoting = true; - pos += quote.length - 1; continue; - }else if(relax_quotes === false){ - const err = this.__error( - new CsvError$1('CSV_INVALID_CLOSING_QUOTE', [ - 'Invalid Closing Quote:', - `got "${String.fromCharCode(nextChr)}"`, - `at line ${this.info.lines}`, - 'instead of delimiter, record delimiter, trimable character', - '(if activated) or comment', - ], this.options, this.__infoField()) - ); - if(err !== undefined) return err; - }else { - this.state.quoting = false; - this.state.wasQuoting = true; - this.state.field.prepend(quote); - pos += quote.length - 1; } - }else { - if(this.state.field.length !== 0){ - // In relax_quotes mode, treat opening quote preceded by chrs as regular - if(relax_quotes === false){ + } + // Not currently escaping and chr is a quote + // TODO: need to compare bytes instead of single char + if(this.state.commenting === false && this.__isQuote(buf, pos)){ + if(this.state.quoting === true){ + const nextChr = buf[pos+quote.length]; + const isNextChrTrimable = rtrim && this.__isCharTrimable(nextChr); + const isNextChrComment = comment !== null && this.__compareBytes(comment, buf, pos+quote.length, nextChr); + const isNextChrDelimiter = this.__isDelimiter(buf, pos+quote.length, nextChr); + const isNextChrRecordDelimiter = record_delimiter.length === 0 ? this.__autoDiscoverRecordDelimiter(buf, pos+quote.length) : this.__isRecordDelimiter(nextChr, buf, pos+quote.length); + // Escape a quote + // Treat next char as a regular character + if(escape !== null && this.__isEscape(buf, pos, chr) && this.__isQuote(buf, pos + escape.length)){ + pos += escape.length - 1; + }else if(!nextChr || isNextChrDelimiter || isNextChrRecordDelimiter || isNextChrComment || isNextChrTrimable){ + this.state.quoting = false; + this.state.wasQuoting = true; + pos += quote.length - 1; + continue; + }else if(relax_quotes === false){ const err = this.__error( - new CsvError$1('INVALID_OPENING_QUOTE', [ - 'Invalid Opening Quote:', - `a quote is found inside a field at line ${this.info.lines}`, - ], this.options, this.__infoField(), { - field: this.state.field, - }) + new CsvError$1('CSV_INVALID_CLOSING_QUOTE', [ + 'Invalid Closing Quote:', + `got "${String.fromCharCode(nextChr)}"`, + `at line ${this.info.lines}`, + 'instead of delimiter, record delimiter, trimable character', + '(if activated) or comment', + ], this.options, this.__infoField()) ); if(err !== undefined) return err; + }else { + this.state.quoting = false; + this.state.wasQuoting = true; + this.state.field.prepend(quote); + pos += quote.length - 1; } }else { - this.state.quoting = true; - pos += quote.length - 1; - continue; - } - } - } - if(this.state.quoting === false){ - const recordDelimiterLength = this.__isRecordDelimiter(chr, buf, pos); - if(recordDelimiterLength !== 0){ - // Do not emit comments which take a full line - const skipCommentLine = this.state.commenting && (this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0); - if(skipCommentLine){ - this.info.comment_lines++; - // Skip full comment line - }else { - // Activate records emition if above from_line - if(this.state.enabled === false && this.info.lines + (this.state.wasRowDelimiter === true ? 1: 0) >= from_line){ - this.state.enabled = true; - this.__resetField(); - this.__resetRecord(); - pos += recordDelimiterLength - 1; + if(this.state.field.length !== 0){ + // In relax_quotes mode, treat opening quote preceded by chrs as regular + if(relax_quotes === false){ + const err = this.__error( + new CsvError$1('INVALID_OPENING_QUOTE', [ + 'Invalid Opening Quote:', + `a quote is found inside a field at line ${this.info.lines}`, + ], this.options, this.__infoField(), { + field: this.state.field, + }) + ); + if(err !== undefined) return err; + } + }else { + this.state.quoting = true; + pos += quote.length - 1; continue; } - // Skip if line is empty and skip_empty_lines activated - if(skip_empty_lines === true && this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0){ - this.info.empty_lines++; - pos += recordDelimiterLength - 1; - continue; + } + } + if(this.state.quoting === false){ + const recordDelimiterLength = this.__isRecordDelimiter(chr, buf, pos); + if(recordDelimiterLength !== 0){ + // Do not emit comments which take a full line + const skipCommentLine = this.state.commenting && (this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0); + if(skipCommentLine){ + this.info.comment_lines++; + // Skip full comment line + }else { + // Activate records emition if above from_line + if(this.state.enabled === false && this.info.lines + (this.state.wasRowDelimiter === true ? 1: 0) >= from_line){ + this.state.enabled = true; + this.__resetField(); + this.__resetRecord(); + pos += recordDelimiterLength - 1; + continue; + } + // Skip if line is empty and skip_empty_lines activated + if(skip_empty_lines === true && this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0){ + this.info.empty_lines++; + pos += recordDelimiterLength - 1; + continue; + } + this.info.bytes = this.state.bufBytesStart + pos; + const errField = this.__onField(); + if(errField !== undefined) return errField; + this.info.bytes = this.state.bufBytesStart + pos + recordDelimiterLength; + const errRecord = this.__onRecord(push); + if(errRecord !== undefined) return errRecord; + if(to !== -1 && this.info.records >= to){ + this.state.stop = true; + close(); + return; + } } + this.state.commenting = false; + pos += recordDelimiterLength - 1; + continue; + } + if(this.state.commenting){ + continue; + } + const commentCount = comment === null ? 0 : this.__compareBytes(comment, buf, pos, chr); + if(commentCount !== 0){ + this.state.commenting = true; + continue; + } + const delimiterLength = this.__isDelimiter(buf, pos, chr); + if(delimiterLength !== 0){ this.info.bytes = this.state.bufBytesStart + pos; const errField = this.__onField(); if(errField !== undefined) return errField; - this.info.bytes = this.state.bufBytesStart + pos + recordDelimiterLength; - const errRecord = this.__onRecord(); - if(errRecord !== undefined) return errRecord; - if(to !== -1 && this.info.records >= to){ - this.state.stop = true; - this.push(null); - return; - } + pos += delimiterLength - 1; + continue; } - this.state.commenting = false; - pos += recordDelimiterLength - 1; - continue; } - if(this.state.commenting){ - continue; - } - const commentCount = comment === null ? 0 : this.__compareBytes(comment, buf, pos, chr); - if(commentCount !== 0){ - this.state.commenting = true; - continue; - } - const delimiterLength = this.__isDelimiter(buf, pos, chr); - if(delimiterLength !== 0){ - this.info.bytes = this.state.bufBytesStart + pos; - const errField = this.__onField(); - if(errField !== undefined) return errField; - pos += delimiterLength - 1; - continue; + } + if(this.state.commenting === false){ + if(max_record_size !== 0 && this.state.record_length + this.state.field.length > max_record_size){ + const err = this.__error( + new CsvError$1('CSV_MAX_RECORD_SIZE', [ + 'Max Record Size:', + 'record exceed the maximum number of tolerated bytes', + `of ${max_record_size}`, + `at line ${this.info.lines}`, + ], this.options, this.__infoField()) + ); + if(err !== undefined) return err; } } - } - if(this.state.commenting === false){ - if(max_record_size !== 0 && this.state.record_length + this.state.field.length > max_record_size){ + const lappend = ltrim === false || this.state.quoting === true || this.state.field.length !== 0 || !this.__isCharTrimable(chr); + // rtrim in non quoting is handle in __onField + const rappend = rtrim === false || this.state.wasQuoting === false; + if(lappend === true && rappend === true){ + this.state.field.append(chr); + }else if(rtrim === true && !this.__isCharTrimable(chr)){ const err = this.__error( - new CsvError$1('CSV_MAX_RECORD_SIZE', [ - 'Max Record Size:', - 'record exceed the maximum number of tolerated bytes', - `of ${max_record_size}`, + new CsvError$1('CSV_NON_TRIMABLE_CHAR_AFTER_CLOSING_QUOTE', [ + 'Invalid Closing Quote:', + 'found non trimable byte after quote', `at line ${this.info.lines}`, ], this.options, this.__infoField()) ); if(err !== undefined) return err; } } - const lappend = ltrim === false || this.state.quoting === true || this.state.field.length !== 0 || !this.__isCharTrimable(chr); - // rtrim in non quoting is handle in __onField - const rappend = rtrim === false || this.state.wasQuoting === false; - if(lappend === true && rappend === true){ - this.state.field.append(chr); - }else if(rtrim === true && !this.__isCharTrimable(chr)){ - const err = this.__error( - new CsvError$1('CSV_NON_TRIMABLE_CHAR_AFTER_CLOSING_QUOTE', [ - 'Invalid Closing Quote:', - 'found non trimable byte after quote', - `at line ${this.info.lines}`, - ], this.options, this.__infoField()) - ); - if(err !== undefined) return err; - } - } - if(end === true){ - // Ensure we are not ending in a quoting state - if(this.state.quoting === true){ - const err = this.__error( - new CsvError$1('CSV_QUOTE_NOT_CLOSED', [ - 'Quote Not Closed:', - `the parsing is finished with an opening quote at line ${this.info.lines}`, - ], this.options, this.__infoField()) - ); - if(err !== undefined) return err; + if(end === true){ + // Ensure we are not ending in a quoting state + if(this.state.quoting === true){ + const err = this.__error( + new CsvError$1('CSV_QUOTE_NOT_CLOSED', [ + 'Quote Not Closed:', + `the parsing is finished with an opening quote at line ${this.info.lines}`, + ], this.options, this.__infoField()) + ); + if(err !== undefined) return err; + }else { + // Skip last line if it has no characters + if(this.state.wasQuoting === true || this.state.record.length !== 0 || this.state.field.length !== 0){ + this.info.bytes = this.state.bufBytesStart + pos; + const errField = this.__onField(); + if(errField !== undefined) return errField; + const errRecord = this.__onRecord(push); + if(errRecord !== undefined) return errRecord; + }else if(this.state.wasRowDelimiter === true){ + this.info.empty_lines++; + }else if(this.state.commenting === true){ + this.info.comment_lines++; + } + } }else { - // Skip last line if it has no characters - if(this.state.wasQuoting === true || this.state.record.length !== 0 || this.state.field.length !== 0){ - this.info.bytes = this.state.bufBytesStart + pos; - const errField = this.__onField(); - if(errField !== undefined) return errField; - const errRecord = this.__onRecord(); - if(errRecord !== undefined) return errRecord; - }else if(this.state.wasRowDelimiter === true){ - this.info.empty_lines++; - }else if(this.state.commenting === true){ - this.info.comment_lines++; + this.state.bufBytesStart += pos; + this.state.previousBuf = buf.slice(pos); + } + if(this.state.wasRowDelimiter === true){ + this.info.lines++; + this.state.wasRowDelimiter = false; + } + }, + __onRecord: function(push){ + const {columns, group_columns_by_name, encoding, info, from, relax_column_count, relax_column_count_less, relax_column_count_more, raw, skip_records_with_empty_values} = this.options; + const {enabled, record} = this.state; + if(enabled === false){ + return this.__resetRecord(); + } + // Convert the first line into column names + const recordLength = record.length; + if(columns === true){ + if(skip_records_with_empty_values === true && isRecordEmpty(record)){ + this.__resetRecord(); + return; + } + return this.__firstLineToColumns(record); + } + if(columns === false && this.info.records === 0){ + this.state.expectedRecordLength = recordLength; + } + if(recordLength !== this.state.expectedRecordLength){ + const err = columns === false ? + new CsvError$1('CSV_RECORD_INCONSISTENT_FIELDS_LENGTH', [ + 'Invalid Record Length:', + `expect ${this.state.expectedRecordLength},`, + `got ${recordLength} on line ${this.info.lines}`, + ], this.options, this.__infoField(), { + record: record, + }) + : + new CsvError$1('CSV_RECORD_INCONSISTENT_COLUMNS', [ + 'Invalid Record Length:', + `columns length is ${columns.length},`, // rename columns + `got ${recordLength} on line ${this.info.lines}`, + ], this.options, this.__infoField(), { + record: record, + }); + if(relax_column_count === true || + (relax_column_count_less === true && recordLength < this.state.expectedRecordLength) || + (relax_column_count_more === true && recordLength > this.state.expectedRecordLength)){ + this.info.invalid_field_length++; + this.state.error = err; + // Error is undefined with skip_records_with_error + }else { + const finalErr = this.__error(err); + if(finalErr) return finalErr; } } - }else { - this.state.bufBytesStart += pos; - this.state.previousBuf = buf.slice(pos); - } - if(this.state.wasRowDelimiter === true){ - this.info.lines++; - this.state.wasRowDelimiter = false; - } - } - __onRecord(){ - const {columns, group_columns_by_name, encoding, info, from, relax_column_count, relax_column_count_less, relax_column_count_more, raw, skip_records_with_empty_values} = this.options; - const {enabled, record} = this.state; - if(enabled === false){ - return this.__resetRecord(); - } - // Convert the first line into column names - const recordLength = record.length; - if(columns === true){ if(skip_records_with_empty_values === true && isRecordEmpty(record)){ this.__resetRecord(); return; } - return this.__firstLineToColumns(record); - } - if(columns === false && this.info.records === 0){ - this.state.expectedRecordLength = recordLength; - } - if(recordLength !== this.state.expectedRecordLength){ - const err = columns === false ? - new CsvError$1('CSV_RECORD_INCONSISTENT_FIELDS_LENGTH', [ - 'Invalid Record Length:', - `expect ${this.state.expectedRecordLength},`, - `got ${recordLength} on line ${this.info.lines}`, - ], this.options, this.__infoField(), { - record: record, - }) - : - new CsvError$1('CSV_RECORD_INCONSISTENT_COLUMNS', [ - 'Invalid Record Length:', - `columns length is ${columns.length},`, // rename columns - `got ${recordLength} on line ${this.info.lines}`, - ], this.options, this.__infoField(), { - record: record, - }); - if(relax_column_count === true || - (relax_column_count_less === true && recordLength < this.state.expectedRecordLength) || - (relax_column_count_more === true && recordLength > this.state.expectedRecordLength)){ - this.info.invalid_field_length++; - this.state.error = err; - // Error is undefined with skip_records_with_error - }else { - const finalErr = this.__error(err); - if(finalErr) return finalErr; + if(this.state.recordHasError === true){ + this.__resetRecord(); + this.state.recordHasError = false; + return; } - } - if(skip_records_with_empty_values === true && isRecordEmpty(record)){ - this.__resetRecord(); - return; - } - if(this.state.recordHasError === true){ - this.__resetRecord(); - this.state.recordHasError = false; - return; - } - this.info.records++; - if(from === 1 || this.info.records >= from){ - const {objname} = this.options; - // With columns, records are object - if(columns !== false){ - const obj = {}; - // Transform record array to an object - for(let i = 0, l = record.length; i < l; i++){ - if(columns[i] === undefined || columns[i].disabled) continue; - // Turn duplicate columns into an array - if (group_columns_by_name === true && obj[columns[i].name] !== undefined) { - if (Array.isArray(obj[columns[i].name])) { - obj[columns[i].name] = obj[columns[i].name].concat(record[i]); + this.info.records++; + if(from === 1 || this.info.records >= from){ + const {objname} = this.options; + // With columns, records are object + if(columns !== false){ + const obj = {}; + // Transform record array to an object + for(let i = 0, l = record.length; i < l; i++){ + if(columns[i] === undefined || columns[i].disabled) continue; + // Turn duplicate columns into an array + if (group_columns_by_name === true && obj[columns[i].name] !== undefined) { + if (Array.isArray(obj[columns[i].name])) { + obj[columns[i].name] = obj[columns[i].name].concat(record[i]); + } else { + obj[columns[i].name] = [obj[columns[i].name], record[i]]; + } } else { - obj[columns[i].name] = [obj[columns[i].name], record[i]]; + obj[columns[i].name] = record[i]; } - } else { - obj[columns[i].name] = record[i]; } - } - // Without objname (default) - if(raw === true || info === true){ - const extRecord = Object.assign( - {record: obj}, - (raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}), - (info === true ? {info: this.__infoRecord()}: {}) - ); - const err = this.__push( - objname === undefined ? extRecord : [obj[objname], extRecord] - ); - if(err){ - return err; - } - }else { - const err = this.__push( - objname === undefined ? obj : [obj[objname], obj] - ); - if(err){ - return err; - } - } - // Without columns, records are array - }else { - if(raw === true || info === true){ - const extRecord = Object.assign( - {record: record}, - raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}, - info === true ? {info: this.__infoRecord()}: {} - ); - const err = this.__push( - objname === undefined ? extRecord : [record[objname], extRecord] - ); - if(err){ - return err; + // Without objname (default) + if(raw === true || info === true){ + const extRecord = Object.assign( + {record: obj}, + (raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}), + (info === true ? {info: this.__infoRecord()}: {}) + ); + const err = this.__push( + objname === undefined ? extRecord : [obj[objname], extRecord] + , push); + if(err){ + return err; + } + }else { + const err = this.__push( + objname === undefined ? obj : [obj[objname], obj] + , push); + if(err){ + return err; + } } + // Without columns, records are array }else { - const err = this.__push( - objname === undefined ? record : [record[objname], record] - ); - if(err){ - return err; + if(raw === true || info === true){ + const extRecord = Object.assign( + {record: record}, + raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}, + info === true ? {info: this.__infoRecord()}: {} + ); + const err = this.__push( + objname === undefined ? extRecord : [record[objname], extRecord] + , push); + if(err){ + return err; + } + }else { + const err = this.__push( + objname === undefined ? record : [record[objname], record] + , push); + if(err){ + return err; + } } } } - } - this.__resetRecord(); - } - __firstLineToColumns(record){ - const {firstLineToHeaders} = this.state; - try{ - const headers = firstLineToHeaders === undefined ? record : firstLineToHeaders.call(null, record); - if(!Array.isArray(headers)){ - return this.__error( - new CsvError$1('CSV_INVALID_COLUMN_MAPPING', [ - 'Invalid Column Mapping:', - 'expect an array from column function,', - `got ${JSON.stringify(headers)}` - ], this.options, this.__infoField(), { - headers: headers, - }) - ); - } - const normalizedHeaders = normalizeColumnsArray(headers); - this.state.expectedRecordLength = normalizedHeaders.length; - this.options.columns = normalizedHeaders; this.__resetRecord(); - return; - }catch(err){ - return err; - } - } - __resetRecord(){ - if(this.options.raw === true){ - this.state.rawBuffer.reset(); - } - this.state.error = undefined; - this.state.record = []; - this.state.record_length = 0; - } - __onField(){ - const {cast, encoding, rtrim, max_record_size} = this.options; - const {enabled, wasQuoting} = this.state; - // Short circuit for the from_line options - if(enabled === false){ - return this.__resetField(); - } - let field = this.state.field.toString(encoding); - if(rtrim === true && wasQuoting === false){ - field = field.trimRight(); - } - if(cast === true){ - const [err, f] = this.__cast(field); - if(err !== undefined) return err; - field = f; - } - this.state.record.push(field); - // Increment record length if record size must not exceed a limit - if(max_record_size !== 0 && typeof field === 'string'){ - this.state.record_length += field.length; - } - this.__resetField(); - } - __resetField(){ - this.state.field.reset(); - this.state.wasQuoting = false; - } - __push(record){ - const {on_record} = this.options; - if(on_record !== undefined){ - const info = this.__infoRecord(); + }, + __firstLineToColumns: function(record){ + const {firstLineToHeaders} = this.state; try{ - record = on_record.call(null, record, info); + const headers = firstLineToHeaders === undefined ? record : firstLineToHeaders.call(null, record); + if(!Array.isArray(headers)){ + return this.__error( + new CsvError$1('CSV_INVALID_COLUMN_MAPPING', [ + 'Invalid Column Mapping:', + 'expect an array from column function,', + `got ${JSON.stringify(headers)}` + ], this.options, this.__infoField(), { + headers: headers, + }) + ); + } + const normalizedHeaders = normalize_columns_array(headers); + this.state.expectedRecordLength = normalizedHeaders.length; + this.options.columns = normalizedHeaders; + this.__resetRecord(); + return; }catch(err){ return err; } - if(record === undefined || record === null){ return; } - } - this.push(record); - } - // Return a tuple with the error and the casted value - __cast(field){ - const {columns, relax_column_count} = this.options; - const isColumns = Array.isArray(columns); - // Dont loose time calling cast - // because the final record is an object - // and this field can't be associated to a key present in columns - if(isColumns === true && relax_column_count && this.options.columns.length <= this.state.record.length){ - return [undefined, undefined]; - } - if(this.state.castField !== null){ - try{ + }, + __resetRecord: function(){ + if(this.options.raw === true){ + this.state.rawBuffer.reset(); + } + this.state.error = undefined; + this.state.record = []; + this.state.record_length = 0; + }, + __onField: function(){ + const {cast, encoding, rtrim, max_record_size} = this.options; + const {enabled, wasQuoting} = this.state; + // Short circuit for the from_line options + if(enabled === false){ + return this.__resetField(); + } + let field = this.state.field.toString(encoding); + if(rtrim === true && wasQuoting === false){ + field = field.trimRight(); + } + if(cast === true){ + const [err, f] = this.__cast(field); + if(err !== undefined) return err; + field = f; + } + this.state.record.push(field); + // Increment record length if record size must not exceed a limit + if(max_record_size !== 0 && typeof field === 'string'){ + this.state.record_length += field.length; + } + this.__resetField(); + }, + __resetField: function(){ + this.state.field.reset(); + this.state.wasQuoting = false; + }, + __push: function(record, push){ + const {on_record} = this.options; + if(on_record !== undefined){ + const info = this.__infoRecord(); + try{ + record = on_record.call(null, record, info); + }catch(err){ + return err; + } + if(record === undefined || record === null){ return; } + } + push(record); + }, + // Return a tuple with the error and the casted value + __cast: function(field){ + const {columns, relax_column_count} = this.options; + const isColumns = Array.isArray(columns); + // Dont loose time calling cast + // because the final record is an object + // and this field can't be associated to a key present in columns + if(isColumns === true && relax_column_count && this.options.columns.length <= this.state.record.length){ + return [undefined, undefined]; + } + if(this.state.castField !== null){ + try{ + const info = this.__infoField(); + return [undefined, this.state.castField.call(null, field, info)]; + }catch(err){ + return [err]; + } + } + if(this.__isFloat(field)){ + return [undefined, parseFloat(field)]; + }else if(this.options.cast_date !== false){ const info = this.__infoField(); - return [undefined, this.state.castField.call(null, field, info)]; - }catch(err){ - return [err]; + return [undefined, this.options.cast_date.call(null, field, info)]; + } + return [undefined, field]; + }, + // Helper to test if a character is a space or a line delimiter + __isCharTrimable: function(chr){ + return chr === space || chr === tab || chr === cr || chr === nl || chr === np; + }, + // Keep it in case we implement the `cast_int` option + // __isInt(value){ + // // return Number.isInteger(parseInt(value)) + // // return !isNaN( parseInt( obj ) ); + // return /^(\-|\+)?[1-9][0-9]*$/.test(value) + // } + __isFloat: function(value){ + return (value - parseFloat(value) + 1) >= 0; // Borrowed from jquery + }, + __compareBytes: function(sourceBuf, targetBuf, targetPos, firstByte){ + if(sourceBuf[0] !== firstByte) return 0; + const sourceLength = sourceBuf.length; + for(let i = 1; i < sourceLength; i++){ + if(sourceBuf[i] !== targetBuf[targetPos+i]) return 0; + } + return sourceLength; + }, + __isDelimiter: function(buf, pos, chr){ + const {delimiter, ignore_last_delimiters} = this.options; + if(ignore_last_delimiters === true && this.state.record.length === this.options.columns.length - 1){ + return 0; + }else if(ignore_last_delimiters !== false && typeof ignore_last_delimiters === 'number' && this.state.record.length === ignore_last_delimiters - 1){ + return 0; + } + loop1: for(let i = 0; i < delimiter.length; i++){ + const del = delimiter[i]; + if(del[0] === chr){ + for(let j = 1; j < del.length; j++){ + if(del[j] !== buf[pos+j]) continue loop1; + } + return del.length; + } } - } - if(this.__isFloat(field)){ - return [undefined, parseFloat(field)]; - }else if(this.options.cast_date !== false){ - const info = this.__infoField(); - return [undefined, this.options.cast_date.call(null, field, info)]; - } - return [undefined, field]; - } - // Helper to test if a character is a space or a line delimiter - __isCharTrimable(chr){ - return chr === space || chr === tab || chr === cr || chr === nl || chr === np; - } - // Keep it in case we implement the `cast_int` option - // __isInt(value){ - // // return Number.isInteger(parseInt(value)) - // // return !isNaN( parseInt( obj ) ); - // return /^(\-|\+)?[1-9][0-9]*$/.test(value) - // } - __isFloat(value){ - return (value - parseFloat(value) + 1) >= 0; // Borrowed from jquery - } - __compareBytes(sourceBuf, targetBuf, targetPos, firstByte){ - if(sourceBuf[0] !== firstByte) return 0; - const sourceLength = sourceBuf.length; - for(let i = 1; i < sourceLength; i++){ - if(sourceBuf[i] !== targetBuf[targetPos+i]) return 0; - } - return sourceLength; - } - __needMoreData(i, bufLen, end){ - if(end) return false; - const {quote} = this.options; - const {quoting, needMoreDataSize, recordDelimiterMaxLength} = this.state; - const numOfCharLeft = bufLen - i - 1; - const requiredLength = Math.max( - needMoreDataSize, - // Skip if the remaining buffer smaller than record delimiter - recordDelimiterMaxLength, - // Skip if the remaining buffer can be record delimiter following the closing quote - // 1 is for quote.length - quoting ? (quote.length + recordDelimiterMaxLength) : 0, - ); - return numOfCharLeft < requiredLength; - } - __isDelimiter(buf, pos, chr){ - const {delimiter, ignore_last_delimiters} = this.options; - if(ignore_last_delimiters === true && this.state.record.length === this.options.columns.length - 1){ - return 0; - }else if(ignore_last_delimiters !== false && typeof ignore_last_delimiters === 'number' && this.state.record.length === ignore_last_delimiters - 1){ return 0; - } - loop1: for(let i = 0; i < delimiter.length; i++){ - const del = delimiter[i]; - if(del[0] === chr){ - for(let j = 1; j < del.length; j++){ - if(del[j] !== buf[pos+j]) continue loop1; + }, + __isRecordDelimiter: function(chr, buf, pos){ + const {record_delimiter} = this.options; + const recordDelimiterLength = record_delimiter.length; + loop1: for(let i = 0; i < recordDelimiterLength; i++){ + const rd = record_delimiter[i]; + const rdLength = rd.length; + if(rd[0] !== chr){ + continue; } - return del.length; - } - } - return 0; - } - __isRecordDelimiter(chr, buf, pos){ - const {record_delimiter} = this.options; - const recordDelimiterLength = record_delimiter.length; - loop1: for(let i = 0; i < recordDelimiterLength; i++){ - const rd = record_delimiter[i]; - const rdLength = rd.length; - if(rd[0] !== chr){ - continue; - } - for(let j = 1; j < rdLength; j++){ - if(rd[j] !== buf[pos+j]){ - continue loop1; + for(let j = 1; j < rdLength; j++){ + if(rd[j] !== buf[pos+j]){ + continue loop1; + } } + return rd.length; } - return rd.length; - } - return 0; - } - __isEscape(buf, pos, chr){ - const {escape} = this.options; - if(escape === null) return false; - const l = escape.length; - if(escape[0] === chr){ + return 0; + }, + __isEscape: function(buf, pos, chr){ + const {escape} = this.options; + if(escape === null) return false; + const l = escape.length; + if(escape[0] === chr){ + for(let i = 0; i < l; i++){ + if(escape[i] !== buf[pos+i]){ + return false; + } + } + return true; + } + return false; + }, + __isQuote: function(buf, pos){ + const {quote} = this.options; + if(quote === null) return false; + const l = quote.length; for(let i = 0; i < l; i++){ - if(escape[i] !== buf[pos+i]){ + if(quote[i] !== buf[pos+i]){ return false; } } return true; - } - return false; - } - __isQuote(buf, pos){ - const {quote} = this.options; - if(quote === null) return false; - const l = quote.length; - for(let i = 0; i < l; i++){ - if(quote[i] !== buf[pos+i]){ - return false; - } - } - return true; - } - __autoDiscoverRecordDelimiter(buf, pos){ - const {encoding} = this.options; - const chr = buf[pos]; - if(chr === cr){ - if(buf[pos+1] === nl){ - this.options.record_delimiter.push(Buffer.from('\r\n', encoding)); - this.state.recordDelimiterMaxLength = 2; - return 2; - }else { - this.options.record_delimiter.push(Buffer.from('\r', encoding)); + }, + __autoDiscoverRecordDelimiter: function(buf, pos){ + const {encoding} = this.options; + const chr = buf[pos]; + if(chr === cr){ + if(buf[pos+1] === nl){ + this.options.record_delimiter.push(Buffer.from('\r\n', encoding)); + this.state.recordDelimiterMaxLength = 2; + return 2; + }else { + this.options.record_delimiter.push(Buffer.from('\r', encoding)); + this.state.recordDelimiterMaxLength = 1; + return 1; + } + }else if(chr === nl){ + this.options.record_delimiter.push(Buffer.from('\n', encoding)); this.state.recordDelimiterMaxLength = 1; return 1; } - }else if(chr === nl){ - this.options.record_delimiter.push(Buffer.from('\n', encoding)); - this.state.recordDelimiterMaxLength = 1; - return 1; - } - return 0; - } - __error(msg){ - const {encoding, raw, skip_records_with_error} = this.options; - const err = typeof msg === 'string' ? new Error(msg) : msg; - if(skip_records_with_error){ - this.state.recordHasError = true; - this.emit('skip', err, raw ? this.state.rawBuffer.toString(encoding) : undefined); - return undefined; - }else { - return err; + return 0; + }, + __error: function(msg){ + const {encoding, raw, skip_records_with_error} = this.options; + const err = typeof msg === 'string' ? new Error(msg) : msg; + if(skip_records_with_error){ + this.state.recordHasError = true; + if(this.options.on_skip !== undefined){ + this.options.on_skip(err, raw ? this.state.rawBuffer.toString(encoding) : undefined); + } + // this.emit('skip', err, raw ? this.state.rawBuffer.toString(encoding) : undefined); + return undefined; + }else { + return err; + } + }, + __infoDataSet: function(){ + return { + ...this.info, + columns: this.options.columns + }; + }, + __infoRecord: function(){ + const {columns, raw, encoding} = this.options; + return { + ...this.__infoDataSet(), + error: this.state.error, + header: columns === true, + index: this.state.record.length, + raw: raw ? this.state.rawBuffer.toString(encoding) : undefined + }; + }, + __infoField: function(){ + const {columns} = this.options; + const isColumns = Array.isArray(columns); + return { + ...this.__infoRecord(), + column: isColumns === true ? + (columns.length > this.state.record.length ? + columns[this.state.record.length].name : + null + ) : + this.state.record.length, + quoting: this.state.wasQuoting, + }; } - } - __infoDataSet(){ - return { - ...this.info, - columns: this.options.columns - }; - } - __infoRecord(){ - const {columns, raw, encoding} = this.options; - return { - ...this.__infoDataSet(), - error: this.state.error, - header: columns === true, - index: this.state.record.length, - raw: raw ? this.state.rawBuffer.toString(encoding) : undefined - }; - } - __infoField(){ - const {columns} = this.options; - const isColumns = Array.isArray(columns); - return { - ...this.__infoRecord(), - column: isColumns === true ? - (columns.length > this.state.record.length ? - columns[this.state.record.length].name : - null - ) : - this.state.record.length, - quoting: this.state.wasQuoting, - }; - } -} + }; +}; -const parse = function(data, options={}){ +const parse = function(data, opts={}){ if(typeof data === 'string'){ data = Buffer.from(data); } - const records = options && options.objname ? {} : []; - const parser = new Parser(options); - parser.push = function(record){ - if(record === null){ - return; - } - if(options.objname === undefined) + const records = opts && opts.objname ? {} : []; + const parser = transform$1(opts); + const push = (record) => { + if(parser.options.objname === undefined) records.push(record); else { records[record[0]] = record[1]; } }; - const err1 = parser.__parse(data, false); + const close = () => {}; + const err1 = parser.parse(data, false, push, close); if(err1 !== undefined) throw err1; - const err2 = parser.__parse(undefined, true); + const err2 = parser.parse(undefined, true, push, close); if(err2 !== undefined) throw err2; return records; }; -const bom_utf8 = Buffer.from([239, 187, 191]); - -class CsvError extends Error { - constructor(code, message, ...contexts) { - if(Array.isArray(message)) message = message.join(' '); - super(message); - if(Error.captureStackTrace !== undefined){ - Error.captureStackTrace(this, CsvError); - } - this.code = code; - for(const context of contexts){ - for(const key in context){ - const value = context[key]; - this[key] = Buffer.isBuffer(value) ? value.toString() : value == null ? value : JSON.parse(JSON.stringify(value)); - } - } - } -} - -const isObject = function(obj){ - return typeof obj === 'object' && obj !== null && ! Array.isArray(obj); -}; - -const underscore = function(str){ - return str.replace(/([A-Z])/g, function(_, match){ - return '_' + match.toLowerCase(); - }); -}; - // Lodash implementation of `get` const charCodeOfDot = '.'.charCodeAt(0); @@ -1624,455 +1616,473 @@ const get = function(object, path){ return (index && index === length) ? object : undefined; }; -class Stringifier extends stream.Transform { - constructor(opts = {}){ - super({...{writableObjectMode: true}, ...opts}); - const options = {}; - let err; - // Merge with user options - for(const opt in opts){ - options[underscore(opt)] = opts[opt]; - } - if((err = this.normalize(options)) !== undefined) throw err; - switch(options.record_delimiter){ - case 'auto': - options.record_delimiter = null; - break; - case 'unix': - options.record_delimiter = "\n"; - break; - case 'mac': - options.record_delimiter = "\r"; - break; - case 'windows': - options.record_delimiter = "\r\n"; - break; - case 'ascii': - options.record_delimiter = "\u001e"; - break; - case 'unicode': - options.record_delimiter = "\u2028"; - break; - } - // Expose options - this.options = options; - // Internal state - this.state = { - stop: false - }; - // Information - this.info = { - records: 0 - }; - } - normalize(options){ - // Normalize option `bom` - if(options.bom === undefined || options.bom === null || options.bom === false){ - options.bom = false; - }else if(options.bom !== true){ - return new CsvError('CSV_OPTION_BOOLEAN_INVALID_TYPE', [ - 'option `bom` is optional and must be a boolean value,', - `got ${JSON.stringify(options.bom)}` - ]); - } - // Normalize option `delimiter` - if(options.delimiter === undefined || options.delimiter === null){ - options.delimiter = ','; - }else if(Buffer.isBuffer(options.delimiter)){ - options.delimiter = options.delimiter.toString(); - }else if(typeof options.delimiter !== 'string'){ - return new CsvError('CSV_OPTION_DELIMITER_INVALID_TYPE', [ - 'option `delimiter` must be a buffer or a string,', - `got ${JSON.stringify(options.delimiter)}` - ]); - } - // Normalize option `quote` - if(options.quote === undefined || options.quote === null){ - options.quote = '"'; - }else if(options.quote === true){ - options.quote = '"'; - }else if(options.quote === false){ - options.quote = ''; - }else if (Buffer.isBuffer(options.quote)){ - options.quote = options.quote.toString(); - }else if(typeof options.quote !== 'string'){ - return new CsvError('CSV_OPTION_QUOTE_INVALID_TYPE', [ - 'option `quote` must be a boolean, a buffer or a string,', - `got ${JSON.stringify(options.quote)}` - ]); - } - // Normalize option `quoted` - if(options.quoted === undefined || options.quoted === null){ - options.quoted = false; - } - // Normalize option `quoted_empty` - if(options.quoted_empty === undefined || options.quoted_empty === null){ - options.quoted_empty = undefined; - } - // Normalize option `quoted_match` - if(options.quoted_match === undefined || options.quoted_match === null || options.quoted_match === false){ - options.quoted_match = null; - }else if(!Array.isArray(options.quoted_match)){ - options.quoted_match = [options.quoted_match]; - } - if(options.quoted_match){ - for(const quoted_match of options.quoted_match){ - const isString = typeof quoted_match === 'string'; - const isRegExp = quoted_match instanceof RegExp; - if(!isString && !isRegExp){ - return Error(`Invalid Option: quoted_match must be a string or a regex, got ${JSON.stringify(quoted_match)}`); +const is_object = function(obj){ + return typeof obj === 'object' && obj !== null && ! Array.isArray(obj); +}; + +const normalize_columns = function(columns){ + if(columns === undefined || columns === null){ + return [undefined, undefined]; + } + if(typeof columns !== 'object'){ + return [Error('Invalid option "columns": expect an array or an object')]; + } + if(!Array.isArray(columns)){ + const newcolumns = []; + for(const k in columns){ + newcolumns.push({ + key: k, + header: columns[k] + }); + } + columns = newcolumns; + }else { + const newcolumns = []; + for(const column of columns){ + if(typeof column === 'string'){ + newcolumns.push({ + key: column, + header: column + }); + }else if(typeof column === 'object' && column !== null && !Array.isArray(column)){ + if(!column.key){ + return [Error('Invalid column definition: property "key" is required')]; + } + if(column.header === undefined){ + column.header = column.key; } + newcolumns.push(column); + }else { + return [Error('Invalid column definition: expect a string or an object')]; } } - // Normalize option `quoted_string` - if(options.quoted_string === undefined || options.quoted_string === null){ - options.quoted_string = false; - } - // Normalize option `eof` - if(options.eof === undefined || options.eof === null){ - options.eof = true; - } - // Normalize option `escape` - if(options.escape === undefined || options.escape === null){ - options.escape = '"'; - }else if(Buffer.isBuffer(options.escape)){ - options.escape = options.escape.toString(); - }else if(typeof options.escape !== 'string'){ - return Error(`Invalid Option: escape must be a buffer or a string, got ${JSON.stringify(options.escape)}`); - } - if (options.escape.length > 1){ - return Error(`Invalid Option: escape must be one character, got ${options.escape.length} characters`); - } - // Normalize option `header` - if(options.header === undefined || options.header === null){ - options.header = false; - } - // Normalize option `columns` - const [err, columns] = this.normalize_columns(options.columns); - if(err) return err; - options.columns = columns; - // Normalize option `quoted` - if(options.quoted === undefined || options.quoted === null){ - options.quoted = false; - } - // Normalize option `cast` - if(options.cast === undefined || options.cast === null){ - options.cast = {}; - } - // Normalize option cast.bigint - if(options.cast.bigint === undefined || options.cast.bigint === null){ - // Cast boolean to string by default - options.cast.bigint = value => '' + value; - } - // Normalize option cast.boolean - if(options.cast.boolean === undefined || options.cast.boolean === null){ - // Cast boolean to string by default - options.cast.boolean = value => value ? '1' : ''; - } - // Normalize option cast.date - if(options.cast.date === undefined || options.cast.date === null){ - // Cast date to timestamp string by default - options.cast.date = value => '' + value.getTime(); - } - // Normalize option cast.number - if(options.cast.number === undefined || options.cast.number === null){ - // Cast number to string using native casting by default - options.cast.number = value => '' + value; - } - // Normalize option cast.object - if(options.cast.object === undefined || options.cast.object === null){ - // Stringify object as JSON by default - options.cast.object = value => JSON.stringify(value); - } - // Normalize option cast.string - if(options.cast.string === undefined || options.cast.string === null){ - // Leave string untouched - options.cast.string = function(value){return value;}; - } - // Normalize option `record_delimiter` - if(options.record_delimiter === undefined || options.record_delimiter === null){ - options.record_delimiter = '\n'; - }else if(Buffer.isBuffer(options.record_delimiter)){ - options.record_delimiter = options.record_delimiter.toString(); - }else if(typeof options.record_delimiter !== 'string'){ - return Error(`Invalid Option: record_delimiter must be a buffer or a string, got ${JSON.stringify(options.record_delimiter)}`); - } - } - _transform(chunk, encoding, callback){ - if(this.state.stop === true){ - return; - } - const err = this.__transform(chunk); - if(err !== undefined){ - this.state.stop = true; - } - callback(err); + columns = newcolumns; } - _flush(callback){ - if(this.state.stop === true){ - // Note, Node.js 12 call flush even after an error, we must prevent - // `callback` from being called in flush without any error. - return; + return [undefined, columns]; +}; + +class CsvError extends Error { + constructor(code, message, ...contexts) { + if(Array.isArray(message)) message = message.join(' '); + super(message); + if(Error.captureStackTrace !== undefined){ + Error.captureStackTrace(this, CsvError); } - if(this.info.records === 0){ - this.bom(); - const err = this.headers(); - if(err) callback(err); + this.code = code; + for(const context of contexts){ + for(const key in context){ + const value = context[key]; + this[key] = Buffer.isBuffer(value) ? value.toString() : value == null ? value : JSON.parse(JSON.stringify(value)); + } } - callback(); } - __transform(chunk){ - // Chunk validation - if(!Array.isArray(chunk) && typeof chunk !== 'object'){ - return Error(`Invalid Record: expect an array or an object, got ${JSON.stringify(chunk)}`); - } - // Detect columns from the first record - if(this.info.records === 0){ - if(Array.isArray(chunk)){ - if(this.options.header === true && this.options.columns === undefined){ - return Error('Undiscoverable Columns: header option requires column option or object records'); +} + +const underscore = function(str){ + return str.replace(/([A-Z])/g, function(_, match){ + return '_' + match.toLowerCase(); + }); +}; + +const normalize_options = function(opts) { + const options = {}; + // Merge with user options + for(const opt in opts){ + options[underscore(opt)] = opts[opt]; + } + // Normalize option `bom` + if(options.bom === undefined || options.bom === null || options.bom === false){ + options.bom = false; + }else if(options.bom !== true){ + return [new CsvError('CSV_OPTION_BOOLEAN_INVALID_TYPE', [ + 'option `bom` is optional and must be a boolean value,', + `got ${JSON.stringify(options.bom)}` + ])]; + } + // Normalize option `delimiter` + if(options.delimiter === undefined || options.delimiter === null){ + options.delimiter = ','; + }else if(Buffer.isBuffer(options.delimiter)){ + options.delimiter = options.delimiter.toString(); + }else if(typeof options.delimiter !== 'string'){ + return [new CsvError('CSV_OPTION_DELIMITER_INVALID_TYPE', [ + 'option `delimiter` must be a buffer or a string,', + `got ${JSON.stringify(options.delimiter)}` + ])]; + } + // Normalize option `quote` + if(options.quote === undefined || options.quote === null){ + options.quote = '"'; + }else if(options.quote === true){ + options.quote = '"'; + }else if(options.quote === false){ + options.quote = ''; + }else if (Buffer.isBuffer(options.quote)){ + options.quote = options.quote.toString(); + }else if(typeof options.quote !== 'string'){ + return [new CsvError('CSV_OPTION_QUOTE_INVALID_TYPE', [ + 'option `quote` must be a boolean, a buffer or a string,', + `got ${JSON.stringify(options.quote)}` + ])]; + } + // Normalize option `quoted` + if(options.quoted === undefined || options.quoted === null){ + options.quoted = false; + } + // Normalize option `quoted_empty` + if(options.quoted_empty === undefined || options.quoted_empty === null){ + options.quoted_empty = undefined; + } + // Normalize option `quoted_match` + if(options.quoted_match === undefined || options.quoted_match === null || options.quoted_match === false){ + options.quoted_match = null; + }else if(!Array.isArray(options.quoted_match)){ + options.quoted_match = [options.quoted_match]; + } + if(options.quoted_match){ + for(const quoted_match of options.quoted_match){ + const isString = typeof quoted_match === 'string'; + const isRegExp = quoted_match instanceof RegExp; + if(!isString && !isRegExp){ + return [Error(`Invalid Option: quoted_match must be a string or a regex, got ${JSON.stringify(quoted_match)}`)]; + } + } + } + // Normalize option `quoted_string` + if(options.quoted_string === undefined || options.quoted_string === null){ + options.quoted_string = false; + } + // Normalize option `eof` + if(options.eof === undefined || options.eof === null){ + options.eof = true; + } + // Normalize option `escape` + if(options.escape === undefined || options.escape === null){ + options.escape = '"'; + }else if(Buffer.isBuffer(options.escape)){ + options.escape = options.escape.toString(); + }else if(typeof options.escape !== 'string'){ + return [Error(`Invalid Option: escape must be a buffer or a string, got ${JSON.stringify(options.escape)}`)]; + } + if (options.escape.length > 1){ + return [Error(`Invalid Option: escape must be one character, got ${options.escape.length} characters`)]; + } + // Normalize option `header` + if(options.header === undefined || options.header === null){ + options.header = false; + } + // Normalize option `columns` + const [errColumns, columns] = normalize_columns(options.columns); + if(errColumns !== undefined) return [errColumns]; + options.columns = columns; + // Normalize option `quoted` + if(options.quoted === undefined || options.quoted === null){ + options.quoted = false; + } + // Normalize option `cast` + if(options.cast === undefined || options.cast === null){ + options.cast = {}; + } + // Normalize option cast.bigint + if(options.cast.bigint === undefined || options.cast.bigint === null){ + // Cast boolean to string by default + options.cast.bigint = value => '' + value; + } + // Normalize option cast.boolean + if(options.cast.boolean === undefined || options.cast.boolean === null){ + // Cast boolean to string by default + options.cast.boolean = value => value ? '1' : ''; + } + // Normalize option cast.date + if(options.cast.date === undefined || options.cast.date === null){ + // Cast date to timestamp string by default + options.cast.date = value => '' + value.getTime(); + } + // Normalize option cast.number + if(options.cast.number === undefined || options.cast.number === null){ + // Cast number to string using native casting by default + options.cast.number = value => '' + value; + } + // Normalize option cast.object + if(options.cast.object === undefined || options.cast.object === null){ + // Stringify object as JSON by default + options.cast.object = value => JSON.stringify(value); + } + // Normalize option cast.string + if(options.cast.string === undefined || options.cast.string === null){ + // Leave string untouched + options.cast.string = function(value){return value;}; + } + // Normalize option `on_record` + if(options.on_record !== undefined && typeof options.on_record !== 'function'){ + return [Error(`Invalid Option: "on_record" must be a function.`)]; + } + // Normalize option `record_delimiter` + if(options.record_delimiter === undefined || options.record_delimiter === null){ + options.record_delimiter = '\n'; + }else if(Buffer.isBuffer(options.record_delimiter)){ + options.record_delimiter = options.record_delimiter.toString(); + }else if(typeof options.record_delimiter !== 'string'){ + return [Error(`Invalid Option: record_delimiter must be a buffer or a string, got ${JSON.stringify(options.record_delimiter)}`)]; + } + switch(options.record_delimiter){ + case 'auto': + options.record_delimiter = null; + break; + case 'unix': + options.record_delimiter = "\n"; + break; + case 'mac': + options.record_delimiter = "\r"; + break; + case 'windows': + options.record_delimiter = "\r\n"; + break; + case 'ascii': + options.record_delimiter = "\u001e"; + break; + case 'unicode': + options.record_delimiter = "\u2028"; + break; + } + return [undefined, options]; +}; + +const bom_utf8 = Buffer.from([239, 187, 191]); + +const stringifier = function(options, state, info){ + return { + options: options, + state: state, + info: info, + __transform: function(chunk, push){ + // Chunk validation + if(!Array.isArray(chunk) && typeof chunk !== 'object'){ + return Error(`Invalid Record: expect an array or an object, got ${JSON.stringify(chunk)}`); + } + // Detect columns from the first record + if(this.info.records === 0){ + if(Array.isArray(chunk)){ + if(this.options.header === true && this.options.columns === undefined){ + return Error('Undiscoverable Columns: header option requires column option or object records'); + } + }else if(this.options.columns === undefined){ + const [err, columns] = normalize_columns(Object.keys(chunk)); + if(err) return; + this.options.columns = columns; } - }else if(this.options.columns === undefined){ - const [err, columns] = this.normalize_columns(Object.keys(chunk)); - if(err) return; - this.options.columns = columns; } - } - // Emit the header - if(this.info.records === 0){ - this.bom(); - const err = this.headers(); - if(err) return err; - } - // Emit and stringify the record if an object or an array - try{ - this.emit('record', chunk, this.info.records); - }catch(err){ - return err; - } - // Convert the record into a string - let err, chunk_string; - if(this.options.eof){ - [err, chunk_string] = this.stringify(chunk); - if(err) return err; - if(chunk_string === undefined){ - return; - }else { - chunk_string = chunk_string + this.options.record_delimiter; + // Emit the header + if(this.info.records === 0){ + this.bom(push); + const err = this.headers(push); + if(err) return err; } - }else { - [err, chunk_string] = this.stringify(chunk); - if(err) return err; - if(chunk_string === undefined){ - return; - }else { - if(this.options.header || this.info.records){ - chunk_string = this.options.record_delimiter + chunk_string; + // Emit and stringify the record if an object or an array + try{ + // this.emit('record', chunk, this.info.records); + if(this.options.on_record){ + this.options.on_record(chunk, this.info.records); } + }catch(err){ + return err; } - } - // Emit the csv - this.info.records++; - this.push(chunk_string); - } - stringify(chunk, chunkIsHeader=false){ - if(typeof chunk !== 'object'){ - return [undefined, chunk]; - } - const {columns} = this.options; - const record = []; - // Record is an array - if(Array.isArray(chunk)){ - // We are getting an array but the user has specified output columns. In - // this case, we respect the columns indexes - if(columns){ - chunk.splice(columns.length); - } - // Cast record elements - for(let i=0; i= 0; - const containsQuote = (quote !== '') && value.indexOf(quote) >= 0; - const containsEscape = value.indexOf(escape) >= 0 && (escape !== quote); - const containsRecordDelimiter = value.indexOf(record_delimiter) >= 0; - const quotedString = quoted_string && typeof field === 'string'; - let quotedMatch = quoted_match && quoted_match.filter(quoted_match => { - if(typeof quoted_match === 'string'){ - return value.indexOf(quoted_match) !== -1; - }else { - return quoted_match.test(value); + } + let csvrecord = ''; + for(let i=0; i 0; - const shouldQuote = containsQuote === true || containsdelimiter || containsRecordDelimiter || quoted || quotedString || quotedMatch; - if(shouldQuote === true && containsEscape === true){ - const regexp = escape === '\\' - ? new RegExp(escape + escape, 'g') - : new RegExp(escape, 'g'); - value = value.replace(regexp, escape + escape); + options = {...this.options, ...options}; + [err, options] = normalize_options(options); + if(err !== undefined){ + return [err]; + } + }else if(value === undefined || value === null){ + options = this.options; + }else { + return [Error(`Invalid Casting Value: returned value must return a string, an object, null or undefined, got ${JSON.stringify(value)}`)]; } - if(containsQuote === true){ - const regexp = new RegExp(quote,'g'); - value = value.replace(regexp, escape + quote); + const {delimiter, escape, quote, quoted, quoted_empty, quoted_string, quoted_match, record_delimiter} = options; + if(value){ + if(typeof value !== 'string'){ + return [Error(`Formatter must return a string, null or undefined, got ${JSON.stringify(value)}`)]; + } + const containsdelimiter = delimiter.length && value.indexOf(delimiter) >= 0; + const containsQuote = (quote !== '') && value.indexOf(quote) >= 0; + const containsEscape = value.indexOf(escape) >= 0 && (escape !== quote); + const containsRecordDelimiter = value.indexOf(record_delimiter) >= 0; + const quotedString = quoted_string && typeof field === 'string'; + let quotedMatch = quoted_match && quoted_match.filter(quoted_match => { + if(typeof quoted_match === 'string'){ + return value.indexOf(quoted_match) !== -1; + }else { + return quoted_match.test(value); + } + }); + quotedMatch = quotedMatch && quotedMatch.length > 0; + const shouldQuote = containsQuote === true || containsdelimiter || containsRecordDelimiter || quoted || quotedString || quotedMatch; + if(shouldQuote === true && containsEscape === true){ + const regexp = escape === '\\' + ? new RegExp(escape + escape, 'g') + : new RegExp(escape, 'g'); + value = value.replace(regexp, escape + escape); + } + if(containsQuote === true){ + const regexp = new RegExp(quote,'g'); + value = value.replace(regexp, escape + quote); + } + if(shouldQuote === true){ + value = quote + value + quote; + } + csvrecord += value; + }else if(quoted_empty === true || (field === '' && quoted_string === true && quoted_empty !== false)){ + csvrecord += quote + quote; } - if(shouldQuote === true){ - value = quote + value + quote; + if(i !== record.length - 1){ + csvrecord += delimiter; } - csvrecord += value; - }else if(quoted_empty === true || (field === '' && quoted_string === true && quoted_empty !== false)){ - csvrecord += quote + quote; } - if(i !== record.length - 1){ - csvrecord += delimiter; + return [undefined, csvrecord]; + }, + bom: function(push){ + if(this.options.bom !== true){ + return; } - } - return [undefined, csvrecord]; - } - bom(){ - if(this.options.bom !== true){ - return; - } - this.push(bom_utf8); - } - headers(){ - if(this.options.header === false){ - return; - } - if(this.options.columns === undefined){ - return; - } - let err; - let headers = this.options.columns.map(column => column.header); - if(this.options.eof){ - [err, headers] = this.stringify(headers, true); - headers += this.options.record_delimiter; - }else { - [err, headers] = this.stringify(headers); - } - if(err) return err; - this.push(headers); - } - __cast(value, context){ - const type = typeof value; - try{ - if(type === 'string'){ // Fine for 99% of the cases - return [undefined, this.options.cast.string(value, context)]; - }else if(type === 'bigint'){ - return [undefined, this.options.cast.bigint(value, context)]; - }else if(type === 'number'){ - return [undefined, this.options.cast.number(value, context)]; - }else if(type === 'boolean'){ - return [undefined, this.options.cast.boolean(value, context)]; - }else if(value instanceof Date){ - return [undefined, this.options.cast.date(value, context)]; - }else if(type === 'object' && value !== null){ - return [undefined, this.options.cast.object(value, context)]; - }else { - return [undefined, value, value]; + push(bom_utf8); + }, + headers: function(push){ + if(this.options.header === false){ + return; } - }catch(err){ - return [err]; - } - } - normalize_columns(columns){ - if(columns === undefined || columns === null){ - return []; - } - if(typeof columns !== 'object'){ - return [Error('Invalid option "columns": expect an array or an object')]; - } - if(!Array.isArray(columns)){ - const newcolumns = []; - for(const k in columns){ - newcolumns.push({ - key: k, - header: columns[k] - }); + if(this.options.columns === undefined){ + return; } - columns = newcolumns; - }else { - const newcolumns = []; - for(const column of columns){ - if(typeof column === 'string'){ - newcolumns.push({ - key: column, - header: column - }); - }else if(typeof column === 'object' && column !== undefined && !Array.isArray(column)){ - if(!column.key){ - return [Error('Invalid column definition: property "key" is required')]; - } - if(column.header === undefined){ - column.header = column.key; - } - newcolumns.push(column); + let err; + let headers = this.options.columns.map(column => column.header); + if(this.options.eof){ + [err, headers] = this.stringify(headers, true); + headers += this.options.record_delimiter; + }else { + [err, headers] = this.stringify(headers); + } + if(err) return err; + push(headers); + }, + __cast: function(value, context){ + const type = typeof value; + try{ + if(type === 'string'){ // Fine for 99% of the cases + return [undefined, this.options.cast.string(value, context)]; + }else if(type === 'bigint'){ + return [undefined, this.options.cast.bigint(value, context)]; + }else if(type === 'number'){ + return [undefined, this.options.cast.number(value, context)]; + }else if(type === 'boolean'){ + return [undefined, this.options.cast.boolean(value, context)]; + }else if(value instanceof Date){ + return [undefined, this.options.cast.date(value, context)]; + }else if(type === 'object' && value !== null){ + return [undefined, this.options.cast.object(value, context)]; }else { - return [Error('Invalid column definition: expect a string or an object')]; + return [undefined, value, value]; } + }catch(err){ + return [err]; } - columns = newcolumns; } - return [undefined, columns]; - } -} + }; +}; -const stringify = function(records, options={}){ +const stringify = function(records, opts={}){ const data = []; - const stringifier = new Stringifier(options); - stringifier.push = function(record){ - if(record === null){ - return; - } - data.push(record.toString()); + const [err, options] = normalize_options(opts); + if(err !== undefined) throw err; + const state = { + stop: false + }; + // Information + const info = { + records: 0 }; + const api = stringifier(options, state, info); + // stringifier.push = function(record){ + // if(record === null){ + // return; + // } + // data.push(record.toString()); + // }; for(const record of records){ - const err = stringifier.__transform(record, null); + const err = api.__transform(record, function(record){ + data.push(record); + }); if(err !== undefined) throw err; } return data.join(''); diff --git a/packages/csv/dist/esm/index.js b/packages/csv/dist/esm/index.js index 6bd538035..28939ae86 100644 --- a/packages/csv/dist/esm/index.js +++ b/packages/csv/dist/esm/index.js @@ -112,7 +112,7 @@ function fromByteArray (uint8) { return parts.join('') } -function read (buffer, offset, isLE, mLen, nBytes) { +function read$1 (buffer, offset, isLE, mLen, nBytes) { var e, m; var eLen = nBytes * 8 - mLen - 1; var eMax = (1 << eLen) - 1; @@ -1393,22 +1393,22 @@ Buffer.prototype.readInt32BE = function readInt32BE (offset, noAssert) { Buffer.prototype.readFloatLE = function readFloatLE (offset, noAssert) { if (!noAssert) checkOffset(offset, 4, this.length); - return read(this, offset, true, 23, 4) + return read$1(this, offset, true, 23, 4) }; Buffer.prototype.readFloatBE = function readFloatBE (offset, noAssert) { if (!noAssert) checkOffset(offset, 4, this.length); - return read(this, offset, false, 23, 4) + return read$1(this, offset, false, 23, 4) }; Buffer.prototype.readDoubleLE = function readDoubleLE (offset, noAssert) { if (!noAssert) checkOffset(offset, 8, this.length); - return read(this, offset, true, 52, 8) + return read$1(this, offset, true, 52, 8) }; Buffer.prototype.readDoubleBE = function readDoubleBE (offset, noAssert) { if (!noAssert) checkOffset(offset, 8, this.length); - return read(this, offset, false, 52, 8) + return read$1(this, offset, false, 52, 8) }; function checkInt (buf, value, offset, ext, max, min) { @@ -2627,7 +2627,7 @@ function format(f) { } }); for (var x = args[i]; i < len; x = args[++i]) { - if (isNull(x) || !isObject$2(x)) { + if (isNull(x) || !isObject(x)) { str += ' ' + x; } else { str += ' ' + inspect(x); @@ -3043,19 +3043,19 @@ function isUndefined(arg) { } function isRegExp(re) { - return isObject$2(re) && objectToString(re) === '[object RegExp]'; + return isObject(re) && objectToString(re) === '[object RegExp]'; } -function isObject$2(arg) { +function isObject(arg) { return typeof arg === 'object' && arg !== null; } function isDate(d) { - return isObject$2(d) && objectToString(d) === '[object Date]'; + return isObject(d) && objectToString(d) === '[object Date]'; } function isError(e) { - return isObject$2(e) && + return isObject(e) && (objectToString(e) === '[object Error]' || e instanceof Error); } @@ -3106,7 +3106,7 @@ function log() { function _extend(origin, add) { // Don't do anything if add isn't an object - if (!add || !isObject$2(add)) return origin; + if (!add || !isObject(add)) return origin; var keys = Object.keys(add); var i = keys.length; @@ -3128,7 +3128,7 @@ var util = { isFunction: isFunction, isError: isError, isDate: isDate, - isObject: isObject$2, + isObject: isObject, isRegExp: isRegExp, isUndefined: isUndefined, isSymbol: isSymbol$1, @@ -5036,20 +5036,64 @@ Stream.prototype.pipe = function(dest, options) { return dest; }; -const Generator = function(options = {}){ +const init_state$1 = (options) => { + // State + return { + start_time: options.duration ? Date.now() : null, + fixed_size_buffer: '', + count_written: 0, + count_created: 0, + }; +}; + +// Generate a random number between 0 and 1 with 2 decimals. The function is idempotent if it detect the "seed" option. +const random = function(options={}){ + if(options.seed){ + return options.seed = options.seed * Math.PI * 100 % 100 / 100; + }else { + return Math.random(); + } +}; + +const types = { + // Generate an ASCII value. + ascii: function({options}){ + const column = []; + const nb_chars = Math.ceil(random(options) * options.maxWordLength); + for(let i=0; i { // Convert Stream Readable options if underscored - if(options.high_water_mark){ - options.highWaterMark = options.high_water_mark; + if(opts.high_water_mark){ + opts.highWaterMark = opts.high_water_mark; } - if(options.object_mode){ - options.objectMode = options.object_mode; + if(opts.object_mode){ + opts.objectMode = opts.object_mode; } - // Call parent constructor - Stream.Readable.call(this, options); // Clone and camelize options - this.options = {}; - for(const k in options){ - this.options[Generator.camelize(k)] = options[k]; + const options = {}; + for(const k in opts){ + options[camelize(k)] = opts[k]; } // Normalize options const dft = { @@ -5067,110 +5111,100 @@ const Generator = function(options = {}){ sleep: 0, }; for(const k in dft){ - if(this.options[k] === undefined){ - this.options[k] = dft[k]; + if(options[k] === undefined){ + options[k] = dft[k]; } } // Default values - if(this.options.eof === true){ - this.options.eof = this.options.rowDelimiter; + if(options.eof === true){ + options.eof = options.rowDelimiter; } - // State - this._ = { - start_time: this.options.duration ? Date.now() : null, - fixed_size_buffer: '', - count_written: 0, - count_created: 0, - }; - if(typeof this.options.columns === 'number'){ - this.options.columns = new Array(this.options.columns); + if(typeof options.columns === 'number'){ + options.columns = new Array(options.columns); } - const accepted_header_types = Object.keys(Generator).filter((t) => (!['super_', 'camelize'].includes(t))); - for(let i = 0; i < this.options.columns.length; i++){ - const v = this.options.columns[i] || 'ascii'; + const accepted_header_types = Object.keys(types).filter((t) => (!['super_', 'camelize'].includes(t))); + for(let i = 0; i < options.columns.length; i++){ + const v = options.columns[i] || 'ascii'; if(typeof v === 'string'){ if(!accepted_header_types.includes(v)){ throw Error(`Invalid column type: got "${v}", default values are ${JSON.stringify(accepted_header_types)}`); } - this.options.columns[i] = Generator[v]; + options.columns[i] = types[v]; } } - return this; + return options; }; -util.inherits(Generator, Stream.Readable); -// Generate a random number between 0 and 1 with 2 decimals. The function is idempotent if it detect the "seed" option. -Generator.prototype.random = function(){ - if(this.options.seed){ - return this.options.seed = this.options.seed * Math.PI * 100 % 100 / 100; - }else { - return Math.random(); - } -}; -// Stop the generation. -Generator.prototype.end = function(){ - this.push(null); -}; -// Put new data into the read queue. -Generator.prototype._read = function(size){ +const read = (options, state, size, push, close) => { // Already started const data = []; - let length = this._.fixed_size_buffer.length; + let length = state.fixed_size_buffer.length; if(length !== 0){ - data.push(this._.fixed_size_buffer); + data.push(state.fixed_size_buffer); } // eslint-disable-next-line while(true){ // Time for some rest: flush first and stop later - if((this._.count_created === this.options.length) || (this.options.end && Date.now() > this.options.end) || (this.options.duration && Date.now() > this._.start_time + this.options.duration)){ + if((state.count_created === options.length) || (options.end && Date.now() > options.end) || (options.duration && Date.now() > state.start_time + options.duration)){ // Flush if(data.length){ - if(this.options.objectMode){ + if(options.objectMode){ for(const record of data){ - this.__push(record); + push(record); } }else { - this.__push(data.join('') + (this.options.eof ? this.options.eof : '')); + push(data.join('') + (options.eof ? options.eof : '')); } - this._.end = true; + state.end = true; }else { - this.push(null); + close(); } return; } // Create the record let record = []; let recordLength; - this.options.columns.forEach((fn) => { - record.push(fn(this)); + options.columns.forEach((fn) => { + const result = fn({options: options, state: state}); + const type = typeof result; + if(result !== null && type !== 'string' && type !== 'number'){ + throw Error([ + 'INVALID_VALUE:', + 'values returned by column function must be', + 'a string, a number or null,', + `got ${JSON.stringify(result)}` + ].join(' ')); + } + record.push(result); }); // Obtain record length - if(this.options.objectMode){ + if(options.objectMode){ recordLength = 0; // recordLength is currently equal to the number of columns // This is wrong and shall equal to 1 record only - for(const column of record) + for(const column of record){ recordLength += column.length; + } }else { // Stringify the record - record = (this._.count_created === 0 ? '' : this.options.rowDelimiter)+record.join(this.options.delimiter); + record = (state.count_created === 0 ? '' : options.rowDelimiter)+record.join(options.delimiter); recordLength = record.length; } - this._.count_created++; + state.count_created++; if(length + recordLength > size){ - if(this.options.objectMode){ + if(options.objectMode){ data.push(record); for(const record of data){ - this.__push(record); + push(record); } }else { - if(this.options.fixedSize){ - this._.fixed_size_buffer = record.substr(size - length); + if(options.fixedSize){ + state.fixed_size_buffer = record.substr(size - length); data.push(record.substr(0, size - length)); }else { data.push(record); } - this.__push(data.join('')); + push(data.join('')); } return; } @@ -5178,42 +5212,41 @@ Generator.prototype._read = function(size){ data.push(record); } }; + +const Generator = function(options = {}){ + this.options = normalize_options$2(options); + // Call parent constructor + Stream.Readable.call(this, this.options); + this.state = init_state$1(this.options); + return this; +}; +util.inherits(Generator, Stream.Readable); + +// Stop the generation. +Generator.prototype.end = function(){ + this.push(null); +}; +// Put new data into the read queue. +Generator.prototype._read = function(size){ + const self = this; + read(this.options, this.state, size, function(chunk) { + self.__push(chunk); + }, function(){ + self.push(null); + }); +}; // Put new data into the read queue. Generator.prototype.__push = function(record){ + // console.log('push', record) const push = () => { - this._.count_written++; + this.state.count_written++; this.push(record); - if(this._.end === true){ + if(this.state.end === true){ return this.push(null); } }; this.options.sleep > 0 ? setTimeout(push, this.options.sleep) : push(); }; -// Generate an ASCII value. -Generator.ascii = function(gen){ - // Column - const column = []; - const nb_chars = Math.ceil(gen.random() * gen.options.maxWordLength); - for(let i=0; i delimiter.length), + // Skip if the remaining buffer can be escape sequence + options.quote !== null ? options.quote.length : 0, + ), + previousBuf: undefined, + quoting: false, + stop: false, + rawBuffer: new ResizeableBuffer(100), + record: [], + recordHasError: false, + record_length: 0, + recordDelimiterMaxLength: options.record_delimiter.length === 0 ? 2 : Math.max(...options.record_delimiter.map((v) => v.length)), + trimChars: [Buffer.from(' ', options.encoding)[0], Buffer.from('\t', options.encoding)[0]], + wasQuoting: false, + wasRowDelimiter: false + }; }; -class CsvError$1 extends Error { - constructor(code, message, options, ...contexts) { - if(Array.isArray(message)) message = message.join(' '); - super(message); - if(Error.captureStackTrace !== undefined){ - Error.captureStackTrace(this, CsvError$1); - } - this.code = code; - for(const context of contexts){ - for(const key in context){ - const value = context[key]; - this[key] = isBuffer$1(value) ? value.toString(options.encoding) : value == null ? value : JSON.parse(JSON.stringify(value)); - } - } - } -} - const underscore$1 = function(str){ return str.replace(/([A-Z])/g, function(_, match){ return '_' + match.toLowerCase(); }); }; -const isObject$1 = function(obj){ - return (typeof obj === 'object' && obj !== null && !Array.isArray(obj)); -}; - -const isRecordEmpty = function(record){ - return record.every((field) => field == null || field.toString && field.toString().trim() === ''); -}; - -const normalizeColumnsArray = function(columns){ - const normalizedColumns = []; - for(let i = 0, l = columns.length; i < l; i++){ - const column = columns[i]; - if(column === undefined || column === null || column === false){ - normalizedColumns[i] = { disabled: true }; - }else if(typeof column === 'string'){ - normalizedColumns[i] = { name: column }; - }else if(isObject$1(column)){ - if(typeof column.name !== 'string'){ - throw new CsvError$1('CSV_OPTION_COLUMNS_MISSING_NAME', [ - 'Option columns missing name:', - `property "name" is required at position ${i}`, - 'when column is an object literal' - ]); - } - normalizedColumns[i] = column; - }else { - throw new CsvError$1('CSV_INVALID_COLUMN_DEFINITION', [ - 'Invalid column definition:', - 'expect a string or a literal object,', - `got ${JSON.stringify(column)} at position ${i}` - ]); - } - } - return normalizedColumns; -}; - -class Parser extends Transform { - constructor(opts = {}){ - super({...{readableObjectMode: true}, ...opts, encoding: null}); - this.__originalOptions = opts; - this.__normalizeOptions(opts); - } - __normalizeOptions(opts){ - const options = {}; - // Merge with user options - for(const opt in opts){ - options[underscore$1(opt)] = opts[opt]; - } - // Normalize option `encoding` - // Note: defined first because other options depends on it - // to convert chars/strings into buffers. - if(options.encoding === undefined || options.encoding === true){ - options.encoding = 'utf8'; - }else if(options.encoding === null || options.encoding === false){ - options.encoding = null; - }else if(typeof options.encoding !== 'string' && options.encoding !== null){ - throw new CsvError$1('CSV_INVALID_OPTION_ENCODING', [ - 'Invalid option encoding:', - 'encoding must be a string or null to return a buffer,', - `got ${JSON.stringify(options.encoding)}` - ], options); - } - // Normalize option `bom` - if(options.bom === undefined || options.bom === null || options.bom === false){ - options.bom = false; - }else if(options.bom !== true){ - throw new CsvError$1('CSV_INVALID_OPTION_BOM', [ - 'Invalid option bom:', 'bom must be true,', - `got ${JSON.stringify(options.bom)}` - ], options); - } - // Normalize option `cast` - let fnCastField = null; - if(options.cast === undefined || options.cast === null || options.cast === false || options.cast === ''){ - options.cast = undefined; - }else if(typeof options.cast === 'function'){ - fnCastField = options.cast; - options.cast = true; - }else if(options.cast !== true){ - throw new CsvError$1('CSV_INVALID_OPTION_CAST', [ - 'Invalid option cast:', 'cast must be true or a function,', - `got ${JSON.stringify(options.cast)}` - ], options); - } - // Normalize option `cast_date` - if(options.cast_date === undefined || options.cast_date === null || options.cast_date === false || options.cast_date === ''){ - options.cast_date = false; - }else if(options.cast_date === true){ - options.cast_date = function(value){ - const date = Date.parse(value); - return !isNaN(date) ? new Date(date) : value; - }; - }else { - throw new CsvError$1('CSV_INVALID_OPTION_CAST_DATE', [ - 'Invalid option cast_date:', 'cast_date must be true or a function,', - `got ${JSON.stringify(options.cast_date)}` - ], options); - } - // Normalize option `columns` - let fnFirstLineToHeaders = null; - if(options.columns === true){ - // Fields in the first line are converted as-is to columns - fnFirstLineToHeaders = undefined; - }else if(typeof options.columns === 'function'){ - fnFirstLineToHeaders = options.columns; - options.columns = true; - }else if(Array.isArray(options.columns)){ - options.columns = normalizeColumnsArray(options.columns); - }else if(options.columns === undefined || options.columns === null || options.columns === false){ - options.columns = false; - }else { - throw new CsvError$1('CSV_INVALID_OPTION_COLUMNS', [ - 'Invalid option columns:', - 'expect an array, a function or true,', - `got ${JSON.stringify(options.columns)}` +const normalize_options$1 = function(opts){ + const options = {}; + // Merge with user options + for(const opt in opts){ + options[underscore$1(opt)] = opts[opt]; + } + // Normalize option `encoding` + // Note: defined first because other options depends on it + // to convert chars/strings into buffers. + if(options.encoding === undefined || options.encoding === true){ + options.encoding = 'utf8'; + }else if(options.encoding === null || options.encoding === false){ + options.encoding = null; + }else if(typeof options.encoding !== 'string' && options.encoding !== null){ + throw new CsvError$1('CSV_INVALID_OPTION_ENCODING', [ + 'Invalid option encoding:', + 'encoding must be a string or null to return a buffer,', + `got ${JSON.stringify(options.encoding)}` + ], options); + } + // Normalize option `bom` + if(options.bom === undefined || options.bom === null || options.bom === false){ + options.bom = false; + }else if(options.bom !== true){ + throw new CsvError$1('CSV_INVALID_OPTION_BOM', [ + 'Invalid option bom:', 'bom must be true,', + `got ${JSON.stringify(options.bom)}` + ], options); + } + // Normalize option `cast` + options.cast_function = null; + if(options.cast === undefined || options.cast === null || options.cast === false || options.cast === ''){ + options.cast = undefined; + }else if(typeof options.cast === 'function'){ + options.cast_function = options.cast; + options.cast = true; + }else if(options.cast !== true){ + throw new CsvError$1('CSV_INVALID_OPTION_CAST', [ + 'Invalid option cast:', 'cast must be true or a function,', + `got ${JSON.stringify(options.cast)}` + ], options); + } + // Normalize option `cast_date` + if(options.cast_date === undefined || options.cast_date === null || options.cast_date === false || options.cast_date === ''){ + options.cast_date = false; + }else if(options.cast_date === true){ + options.cast_date = function(value){ + const date = Date.parse(value); + return !isNaN(date) ? new Date(date) : value; + }; + }else { + throw new CsvError$1('CSV_INVALID_OPTION_CAST_DATE', [ + 'Invalid option cast_date:', 'cast_date must be true or a function,', + `got ${JSON.stringify(options.cast_date)}` + ], options); + } + // Normalize option `columns` + options.cast_first_line_to_header = null; + if(options.columns === true){ + // Fields in the first line are converted as-is to columns + options.cast_first_line_to_header = undefined; + }else if(typeof options.columns === 'function'){ + options.cast_first_line_to_header = options.columns; + options.columns = true; + }else if(Array.isArray(options.columns)){ + options.columns = normalize_columns_array(options.columns); + }else if(options.columns === undefined || options.columns === null || options.columns === false){ + options.columns = false; + }else { + throw new CsvError$1('CSV_INVALID_OPTION_COLUMNS', [ + 'Invalid option columns:', + 'expect an array, a function or true,', + `got ${JSON.stringify(options.columns)}` + ], options); + } + // Normalize option `group_columns_by_name` + if(options.group_columns_by_name === undefined || options.group_columns_by_name === null || options.group_columns_by_name === false){ + options.group_columns_by_name = false; + }else if(options.group_columns_by_name !== true){ + throw new CsvError$1('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ + 'Invalid option group_columns_by_name:', + 'expect an boolean,', + `got ${JSON.stringify(options.group_columns_by_name)}` + ], options); + }else if(options.columns === false){ + throw new CsvError$1('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ + 'Invalid option group_columns_by_name:', + 'the `columns` mode must be activated.' + ], options); + } + // Normalize option `comment` + if(options.comment === undefined || options.comment === null || options.comment === false || options.comment === ''){ + options.comment = null; + }else { + if(typeof options.comment === 'string'){ + options.comment = Buffer.from(options.comment, options.encoding); + } + if(!isBuffer$1(options.comment)){ + throw new CsvError$1('CSV_INVALID_OPTION_COMMENT', [ + 'Invalid option comment:', + 'comment must be a buffer or a string,', + `got ${JSON.stringify(options.comment)}` ], options); } - // Normalize option `group_columns_by_name` - if(options.group_columns_by_name === undefined || options.group_columns_by_name === null || options.group_columns_by_name === false){ - options.group_columns_by_name = false; - }else if(options.group_columns_by_name !== true){ - throw new CsvError$1('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ - 'Invalid option group_columns_by_name:', - 'expect an boolean,', - `got ${JSON.stringify(options.group_columns_by_name)}` - ], options); - }else if(options.columns === false){ - throw new CsvError$1('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ - 'Invalid option group_columns_by_name:', - 'the `columns` mode must be activated.' - ], options); + } + // Normalize option `delimiter` + const delimiter_json = JSON.stringify(options.delimiter); + if(!Array.isArray(options.delimiter)) options.delimiter = [options.delimiter]; + if(options.delimiter.length === 0){ + throw new CsvError$1('CSV_INVALID_OPTION_DELIMITER', [ + 'Invalid option delimiter:', + 'delimiter must be a non empty string or buffer or array of string|buffer,', + `got ${delimiter_json}` + ], options); + } + options.delimiter = options.delimiter.map(function(delimiter){ + if(delimiter === undefined || delimiter === null || delimiter === false){ + return Buffer.from(',', options.encoding); } - // Normalize option `comment` - if(options.comment === undefined || options.comment === null || options.comment === false || options.comment === ''){ - options.comment = null; - }else { - if(typeof options.comment === 'string'){ - options.comment = Buffer.from(options.comment, options.encoding); - } - if(!isBuffer$1(options.comment)){ - throw new CsvError$1('CSV_INVALID_OPTION_COMMENT', [ - 'Invalid option comment:', - 'comment must be a buffer or a string,', - `got ${JSON.stringify(options.comment)}` - ], options); - } + if(typeof delimiter === 'string'){ + delimiter = Buffer.from(delimiter, options.encoding); } - // Normalize option `delimiter` - const delimiter_json = JSON.stringify(options.delimiter); - if(!Array.isArray(options.delimiter)) options.delimiter = [options.delimiter]; - if(options.delimiter.length === 0){ + if(!isBuffer$1(delimiter) || delimiter.length === 0){ throw new CsvError$1('CSV_INVALID_OPTION_DELIMITER', [ 'Invalid option delimiter:', 'delimiter must be a non empty string or buffer or array of string|buffer,', `got ${delimiter_json}` ], options); } - options.delimiter = options.delimiter.map(function(delimiter){ - if(delimiter === undefined || delimiter === null || delimiter === false){ - return Buffer.from(',', options.encoding); - } - if(typeof delimiter === 'string'){ - delimiter = Buffer.from(delimiter, options.encoding); - } - if(!isBuffer$1(delimiter) || delimiter.length === 0){ - throw new CsvError$1('CSV_INVALID_OPTION_DELIMITER', [ - 'Invalid option delimiter:', - 'delimiter must be a non empty string or buffer or array of string|buffer,', - `got ${delimiter_json}` - ], options); - } - return delimiter; - }); - // Normalize option `escape` - if(options.escape === undefined || options.escape === true){ - options.escape = Buffer.from('"', options.encoding); - }else if(typeof options.escape === 'string'){ - options.escape = Buffer.from(options.escape, options.encoding); - }else if (options.escape === null || options.escape === false){ - options.escape = null; - } - if(options.escape !== null){ - if(!isBuffer$1(options.escape)){ - throw new Error(`Invalid Option: escape must be a buffer, a string or a boolean, got ${JSON.stringify(options.escape)}`); - } + return delimiter; + }); + // Normalize option `escape` + if(options.escape === undefined || options.escape === true){ + options.escape = Buffer.from('"', options.encoding); + }else if(typeof options.escape === 'string'){ + options.escape = Buffer.from(options.escape, options.encoding); + }else if (options.escape === null || options.escape === false){ + options.escape = null; + } + if(options.escape !== null){ + if(!isBuffer$1(options.escape)){ + throw new Error(`Invalid Option: escape must be a buffer, a string or a boolean, got ${JSON.stringify(options.escape)}`); + } + } + // Normalize option `from` + if(options.from === undefined || options.from === null){ + options.from = 1; + }else { + if(typeof options.from === 'string' && /\d+/.test(options.from)){ + options.from = parseInt(options.from); } - // Normalize option `from` - if(options.from === undefined || options.from === null){ - options.from = 1; - }else { - if(typeof options.from === 'string' && /\d+/.test(options.from)){ - options.from = parseInt(options.from); - } - if(Number.isInteger(options.from)){ - if(options.from < 0){ - throw new Error(`Invalid Option: from must be a positive integer, got ${JSON.stringify(opts.from)}`); - } - }else { - throw new Error(`Invalid Option: from must be an integer, got ${JSON.stringify(options.from)}`); + if(Number.isInteger(options.from)){ + if(options.from < 0){ + throw new Error(`Invalid Option: from must be a positive integer, got ${JSON.stringify(opts.from)}`); } - } - // Normalize option `from_line` - if(options.from_line === undefined || options.from_line === null){ - options.from_line = 1; }else { - if(typeof options.from_line === 'string' && /\d+/.test(options.from_line)){ - options.from_line = parseInt(options.from_line); - } - if(Number.isInteger(options.from_line)){ - if(options.from_line <= 0){ - throw new Error(`Invalid Option: from_line must be a positive integer greater than 0, got ${JSON.stringify(opts.from_line)}`); - } - }else { - throw new Error(`Invalid Option: from_line must be an integer, got ${JSON.stringify(opts.from_line)}`); - } - } - // Normalize options `ignore_last_delimiters` - if(options.ignore_last_delimiters === undefined || options.ignore_last_delimiters === null){ - options.ignore_last_delimiters = false; - }else if(typeof options.ignore_last_delimiters === 'number'){ - options.ignore_last_delimiters = Math.floor(options.ignore_last_delimiters); - if(options.ignore_last_delimiters === 0){ - options.ignore_last_delimiters = false; - } - }else if(typeof options.ignore_last_delimiters !== 'boolean'){ - throw new CsvError$1('CSV_INVALID_OPTION_IGNORE_LAST_DELIMITERS', [ - 'Invalid option `ignore_last_delimiters`:', - 'the value must be a boolean value or an integer,', - `got ${JSON.stringify(options.ignore_last_delimiters)}` - ], options); + throw new Error(`Invalid Option: from must be an integer, got ${JSON.stringify(options.from)}`); } - if(options.ignore_last_delimiters === true && options.columns === false){ - throw new CsvError$1('CSV_IGNORE_LAST_DELIMITERS_REQUIRES_COLUMNS', [ - 'The option `ignore_last_delimiters`', - 'requires the activation of the `columns` option' - ], options); + } + // Normalize option `from_line` + if(options.from_line === undefined || options.from_line === null){ + options.from_line = 1; + }else { + if(typeof options.from_line === 'string' && /\d+/.test(options.from_line)){ + options.from_line = parseInt(options.from_line); } - // Normalize option `info` - if(options.info === undefined || options.info === null || options.info === false){ - options.info = false; - }else if(options.info !== true){ - throw new Error(`Invalid Option: info must be true, got ${JSON.stringify(options.info)}`); - } - // Normalize option `max_record_size` - if(options.max_record_size === undefined || options.max_record_size === null || options.max_record_size === false){ - options.max_record_size = 0; - }else if(Number.isInteger(options.max_record_size) && options.max_record_size >= 0);else if(typeof options.max_record_size === 'string' && /\d+/.test(options.max_record_size)){ - options.max_record_size = parseInt(options.max_record_size); - }else { - throw new Error(`Invalid Option: max_record_size must be a positive integer, got ${JSON.stringify(options.max_record_size)}`); - } - // Normalize option `objname` - if(options.objname === undefined || options.objname === null || options.objname === false){ - options.objname = undefined; - }else if(isBuffer$1(options.objname)){ - if(options.objname.length === 0){ - throw new Error(`Invalid Option: objname must be a non empty buffer`); - } - if(options.encoding === null);else { - options.objname = options.objname.toString(options.encoding); - } - }else if(typeof options.objname === 'string'){ - if(options.objname.length === 0){ - throw new Error(`Invalid Option: objname must be a non empty string`); - } - // Great, nothing to do - }else if(typeof options.objname === 'number');else { - throw new Error(`Invalid Option: objname must be a string or a buffer, got ${options.objname}`); - } - if(options.objname !== undefined){ - if(typeof options.objname === 'number'){ - if(options.columns !== false){ - throw Error('Invalid Option: objname index cannot be combined with columns or be defined as a field'); - } - }else { // A string or a buffer - if(options.columns === false){ - throw Error('Invalid Option: objname field must be combined with columns or be defined as an index'); - } + if(Number.isInteger(options.from_line)){ + if(options.from_line <= 0){ + throw new Error(`Invalid Option: from_line must be a positive integer greater than 0, got ${JSON.stringify(opts.from_line)}`); } + }else { + throw new Error(`Invalid Option: from_line must be an integer, got ${JSON.stringify(opts.from_line)}`); } - // Normalize option `on_record` - if(options.on_record === undefined || options.on_record === null){ - options.on_record = undefined; - }else if(typeof options.on_record !== 'function'){ - throw new CsvError$1('CSV_INVALID_OPTION_ON_RECORD', [ - 'Invalid option `on_record`:', - 'expect a function,', - `got ${JSON.stringify(options.on_record)}` - ], options); + } + // Normalize options `ignore_last_delimiters` + if(options.ignore_last_delimiters === undefined || options.ignore_last_delimiters === null){ + options.ignore_last_delimiters = false; + }else if(typeof options.ignore_last_delimiters === 'number'){ + options.ignore_last_delimiters = Math.floor(options.ignore_last_delimiters); + if(options.ignore_last_delimiters === 0){ + options.ignore_last_delimiters = false; } - // Normalize option `quote` - if(options.quote === null || options.quote === false || options.quote === ''){ - options.quote = null; - }else { - if(options.quote === undefined || options.quote === true){ - options.quote = Buffer.from('"', options.encoding); - }else if(typeof options.quote === 'string'){ - options.quote = Buffer.from(options.quote, options.encoding); + }else if(typeof options.ignore_last_delimiters !== 'boolean'){ + throw new CsvError$1('CSV_INVALID_OPTION_IGNORE_LAST_DELIMITERS', [ + 'Invalid option `ignore_last_delimiters`:', + 'the value must be a boolean value or an integer,', + `got ${JSON.stringify(options.ignore_last_delimiters)}` + ], options); + } + if(options.ignore_last_delimiters === true && options.columns === false){ + throw new CsvError$1('CSV_IGNORE_LAST_DELIMITERS_REQUIRES_COLUMNS', [ + 'The option `ignore_last_delimiters`', + 'requires the activation of the `columns` option' + ], options); + } + // Normalize option `info` + if(options.info === undefined || options.info === null || options.info === false){ + options.info = false; + }else if(options.info !== true){ + throw new Error(`Invalid Option: info must be true, got ${JSON.stringify(options.info)}`); + } + // Normalize option `max_record_size` + if(options.max_record_size === undefined || options.max_record_size === null || options.max_record_size === false){ + options.max_record_size = 0; + }else if(Number.isInteger(options.max_record_size) && options.max_record_size >= 0);else if(typeof options.max_record_size === 'string' && /\d+/.test(options.max_record_size)){ + options.max_record_size = parseInt(options.max_record_size); + }else { + throw new Error(`Invalid Option: max_record_size must be a positive integer, got ${JSON.stringify(options.max_record_size)}`); + } + // Normalize option `objname` + if(options.objname === undefined || options.objname === null || options.objname === false){ + options.objname = undefined; + }else if(isBuffer$1(options.objname)){ + if(options.objname.length === 0){ + throw new Error(`Invalid Option: objname must be a non empty buffer`); + } + if(options.encoding === null);else { + options.objname = options.objname.toString(options.encoding); + } + }else if(typeof options.objname === 'string'){ + if(options.objname.length === 0){ + throw new Error(`Invalid Option: objname must be a non empty string`); + } + // Great, nothing to do + }else if(typeof options.objname === 'number');else { + throw new Error(`Invalid Option: objname must be a string or a buffer, got ${options.objname}`); + } + if(options.objname !== undefined){ + if(typeof options.objname === 'number'){ + if(options.columns !== false){ + throw Error('Invalid Option: objname index cannot be combined with columns or be defined as a field'); } - if(!isBuffer$1(options.quote)){ - throw new Error(`Invalid Option: quote must be a buffer or a string, got ${JSON.stringify(options.quote)}`); + }else { // A string or a buffer + if(options.columns === false){ + throw Error('Invalid Option: objname field must be combined with columns or be defined as an index'); } } - // Normalize option `raw` - if(options.raw === undefined || options.raw === null || options.raw === false){ - options.raw = false; - }else if(options.raw !== true){ - throw new Error(`Invalid Option: raw must be true, got ${JSON.stringify(options.raw)}`); - } - // Normalize option `record_delimiter` - if(options.record_delimiter === undefined){ - options.record_delimiter = []; - }else if(typeof options.record_delimiter === 'string' || isBuffer$1(options.record_delimiter)){ - if(options.record_delimiter.length === 0){ - throw new CsvError$1('CSV_INVALID_OPTION_RECORD_DELIMITER', [ - 'Invalid option `record_delimiter`:', - 'value must be a non empty string or buffer,', - `got ${JSON.stringify(options.record_delimiter)}` - ], options); - } - options.record_delimiter = [options.record_delimiter]; - }else if(!Array.isArray(options.record_delimiter)){ + } + // Normalize option `on_record` + if(options.on_record === undefined || options.on_record === null){ + options.on_record = undefined; + }else if(typeof options.on_record !== 'function'){ + throw new CsvError$1('CSV_INVALID_OPTION_ON_RECORD', [ + 'Invalid option `on_record`:', + 'expect a function,', + `got ${JSON.stringify(options.on_record)}` + ], options); + } + // Normalize option `quote` + if(options.quote === null || options.quote === false || options.quote === ''){ + options.quote = null; + }else { + if(options.quote === undefined || options.quote === true){ + options.quote = Buffer.from('"', options.encoding); + }else if(typeof options.quote === 'string'){ + options.quote = Buffer.from(options.quote, options.encoding); + } + if(!isBuffer$1(options.quote)){ + throw new Error(`Invalid Option: quote must be a buffer or a string, got ${JSON.stringify(options.quote)}`); + } + } + // Normalize option `raw` + if(options.raw === undefined || options.raw === null || options.raw === false){ + options.raw = false; + }else if(options.raw !== true){ + throw new Error(`Invalid Option: raw must be true, got ${JSON.stringify(options.raw)}`); + } + // Normalize option `record_delimiter` + if(options.record_delimiter === undefined){ + options.record_delimiter = []; + }else if(typeof options.record_delimiter === 'string' || isBuffer$1(options.record_delimiter)){ + if(options.record_delimiter.length === 0){ throw new CsvError$1('CSV_INVALID_OPTION_RECORD_DELIMITER', [ 'Invalid option `record_delimiter`:', - 'value must be a string, a buffer or array of string|buffer,', + 'value must be a non empty string or buffer,', `got ${JSON.stringify(options.record_delimiter)}` ], options); } - options.record_delimiter = options.record_delimiter.map(function(rd, i){ - if(typeof rd !== 'string' && ! isBuffer$1(rd)){ - throw new CsvError$1('CSV_INVALID_OPTION_RECORD_DELIMITER', [ - 'Invalid option `record_delimiter`:', - 'value must be a string, a buffer or array of string|buffer', - `at index ${i},`, - `got ${JSON.stringify(rd)}` - ], options); - }else if(rd.length === 0){ - throw new CsvError$1('CSV_INVALID_OPTION_RECORD_DELIMITER', [ - 'Invalid option `record_delimiter`:', - 'value must be a non empty string or buffer', - `at index ${i},`, - `got ${JSON.stringify(rd)}` - ], options); - } - if(typeof rd === 'string'){ - rd = Buffer.from(rd, options.encoding); - } - return rd; - }); - // Normalize option `relax_column_count` - if(typeof options.relax_column_count === 'boolean');else if(options.relax_column_count === undefined || options.relax_column_count === null){ - options.relax_column_count = false; - }else { - throw new Error(`Invalid Option: relax_column_count must be a boolean, got ${JSON.stringify(options.relax_column_count)}`); - } - if(typeof options.relax_column_count_less === 'boolean');else if(options.relax_column_count_less === undefined || options.relax_column_count_less === null){ - options.relax_column_count_less = false; - }else { - throw new Error(`Invalid Option: relax_column_count_less must be a boolean, got ${JSON.stringify(options.relax_column_count_less)}`); - } - if(typeof options.relax_column_count_more === 'boolean');else if(options.relax_column_count_more === undefined || options.relax_column_count_more === null){ - options.relax_column_count_more = false; - }else { - throw new Error(`Invalid Option: relax_column_count_more must be a boolean, got ${JSON.stringify(options.relax_column_count_more)}`); - } - // Normalize option `relax_quotes` - if(typeof options.relax_quotes === 'boolean');else if(options.relax_quotes === undefined || options.relax_quotes === null){ - options.relax_quotes = false; - }else { - throw new Error(`Invalid Option: relax_quotes must be a boolean, got ${JSON.stringify(options.relax_quotes)}`); + options.record_delimiter = [options.record_delimiter]; + }else if(!Array.isArray(options.record_delimiter)){ + throw new CsvError$1('CSV_INVALID_OPTION_RECORD_DELIMITER', [ + 'Invalid option `record_delimiter`:', + 'value must be a string, a buffer or array of string|buffer,', + `got ${JSON.stringify(options.record_delimiter)}` + ], options); + } + options.record_delimiter = options.record_delimiter.map(function(rd, i){ + if(typeof rd !== 'string' && ! isBuffer$1(rd)){ + throw new CsvError$1('CSV_INVALID_OPTION_RECORD_DELIMITER', [ + 'Invalid option `record_delimiter`:', + 'value must be a string, a buffer or array of string|buffer', + `at index ${i},`, + `got ${JSON.stringify(rd)}` + ], options); + }else if(rd.length === 0){ + throw new CsvError$1('CSV_INVALID_OPTION_RECORD_DELIMITER', [ + 'Invalid option `record_delimiter`:', + 'value must be a non empty string or buffer', + `at index ${i},`, + `got ${JSON.stringify(rd)}` + ], options); } - // Normalize option `skip_empty_lines` - if(typeof options.skip_empty_lines === 'boolean');else if(options.skip_empty_lines === undefined || options.skip_empty_lines === null){ - options.skip_empty_lines = false; - }else { - throw new Error(`Invalid Option: skip_empty_lines must be a boolean, got ${JSON.stringify(options.skip_empty_lines)}`); + if(typeof rd === 'string'){ + rd = Buffer.from(rd, options.encoding); } - // Normalize option `skip_records_with_empty_values` - if(typeof options.skip_records_with_empty_values === 'boolean');else if(options.skip_records_with_empty_values === undefined || options.skip_records_with_empty_values === null){ - options.skip_records_with_empty_values = false; - }else { - throw new Error(`Invalid Option: skip_records_with_empty_values must be a boolean, got ${JSON.stringify(options.skip_records_with_empty_values)}`); + return rd; + }); + // Normalize option `relax_column_count` + if(typeof options.relax_column_count === 'boolean');else if(options.relax_column_count === undefined || options.relax_column_count === null){ + options.relax_column_count = false; + }else { + throw new Error(`Invalid Option: relax_column_count must be a boolean, got ${JSON.stringify(options.relax_column_count)}`); + } + if(typeof options.relax_column_count_less === 'boolean');else if(options.relax_column_count_less === undefined || options.relax_column_count_less === null){ + options.relax_column_count_less = false; + }else { + throw new Error(`Invalid Option: relax_column_count_less must be a boolean, got ${JSON.stringify(options.relax_column_count_less)}`); + } + if(typeof options.relax_column_count_more === 'boolean');else if(options.relax_column_count_more === undefined || options.relax_column_count_more === null){ + options.relax_column_count_more = false; + }else { + throw new Error(`Invalid Option: relax_column_count_more must be a boolean, got ${JSON.stringify(options.relax_column_count_more)}`); + } + // Normalize option `relax_quotes` + if(typeof options.relax_quotes === 'boolean');else if(options.relax_quotes === undefined || options.relax_quotes === null){ + options.relax_quotes = false; + }else { + throw new Error(`Invalid Option: relax_quotes must be a boolean, got ${JSON.stringify(options.relax_quotes)}`); + } + // Normalize option `skip_empty_lines` + if(typeof options.skip_empty_lines === 'boolean');else if(options.skip_empty_lines === undefined || options.skip_empty_lines === null){ + options.skip_empty_lines = false; + }else { + throw new Error(`Invalid Option: skip_empty_lines must be a boolean, got ${JSON.stringify(options.skip_empty_lines)}`); + } + // Normalize option `skip_records_with_empty_values` + if(typeof options.skip_records_with_empty_values === 'boolean');else if(options.skip_records_with_empty_values === undefined || options.skip_records_with_empty_values === null){ + options.skip_records_with_empty_values = false; + }else { + throw new Error(`Invalid Option: skip_records_with_empty_values must be a boolean, got ${JSON.stringify(options.skip_records_with_empty_values)}`); + } + // Normalize option `skip_records_with_error` + if(typeof options.skip_records_with_error === 'boolean');else if(options.skip_records_with_error === undefined || options.skip_records_with_error === null){ + options.skip_records_with_error = false; + }else { + throw new Error(`Invalid Option: skip_records_with_error must be a boolean, got ${JSON.stringify(options.skip_records_with_error)}`); + } + // Normalize option `rtrim` + if(options.rtrim === undefined || options.rtrim === null || options.rtrim === false){ + options.rtrim = false; + }else if(options.rtrim !== true){ + throw new Error(`Invalid Option: rtrim must be a boolean, got ${JSON.stringify(options.rtrim)}`); + } + // Normalize option `ltrim` + if(options.ltrim === undefined || options.ltrim === null || options.ltrim === false){ + options.ltrim = false; + }else if(options.ltrim !== true){ + throw new Error(`Invalid Option: ltrim must be a boolean, got ${JSON.stringify(options.ltrim)}`); + } + // Normalize option `trim` + if(options.trim === undefined || options.trim === null || options.trim === false){ + options.trim = false; + }else if(options.trim !== true){ + throw new Error(`Invalid Option: trim must be a boolean, got ${JSON.stringify(options.trim)}`); + } + // Normalize options `trim`, `ltrim` and `rtrim` + if(options.trim === true && opts.ltrim !== false){ + options.ltrim = true; + }else if(options.ltrim !== true){ + options.ltrim = false; + } + if(options.trim === true && opts.rtrim !== false){ + options.rtrim = true; + }else if(options.rtrim !== true){ + options.rtrim = false; + } + // Normalize option `to` + if(options.to === undefined || options.to === null){ + options.to = -1; + }else { + if(typeof options.to === 'string' && /\d+/.test(options.to)){ + options.to = parseInt(options.to); } - // Normalize option `skip_records_with_error` - if(typeof options.skip_records_with_error === 'boolean');else if(options.skip_records_with_error === undefined || options.skip_records_with_error === null){ - options.skip_records_with_error = false; - }else { - throw new Error(`Invalid Option: skip_records_with_error must be a boolean, got ${JSON.stringify(options.skip_records_with_error)}`); - } - // Normalize option `rtrim` - if(options.rtrim === undefined || options.rtrim === null || options.rtrim === false){ - options.rtrim = false; - }else if(options.rtrim !== true){ - throw new Error(`Invalid Option: rtrim must be a boolean, got ${JSON.stringify(options.rtrim)}`); - } - // Normalize option `ltrim` - if(options.ltrim === undefined || options.ltrim === null || options.ltrim === false){ - options.ltrim = false; - }else if(options.ltrim !== true){ - throw new Error(`Invalid Option: ltrim must be a boolean, got ${JSON.stringify(options.ltrim)}`); - } - // Normalize option `trim` - if(options.trim === undefined || options.trim === null || options.trim === false){ - options.trim = false; - }else if(options.trim !== true){ - throw new Error(`Invalid Option: trim must be a boolean, got ${JSON.stringify(options.trim)}`); - } - // Normalize options `trim`, `ltrim` and `rtrim` - if(options.trim === true && opts.ltrim !== false){ - options.ltrim = true; - }else if(options.ltrim !== true){ - options.ltrim = false; - } - if(options.trim === true && opts.rtrim !== false){ - options.rtrim = true; - }else if(options.rtrim !== true){ - options.rtrim = false; - } - // Normalize option `to` - if(options.to === undefined || options.to === null){ - options.to = -1; - }else { - if(typeof options.to === 'string' && /\d+/.test(options.to)){ - options.to = parseInt(options.to); + if(Number.isInteger(options.to)){ + if(options.to <= 0){ + throw new Error(`Invalid Option: to must be a positive integer greater than 0, got ${JSON.stringify(opts.to)}`); } - if(Number.isInteger(options.to)){ - if(options.to <= 0){ - throw new Error(`Invalid Option: to must be a positive integer greater than 0, got ${JSON.stringify(opts.to)}`); - } - }else { - throw new Error(`Invalid Option: to must be an integer, got ${JSON.stringify(opts.to)}`); - } - } - // Normalize option `to_line` - if(options.to_line === undefined || options.to_line === null){ - options.to_line = -1; }else { - if(typeof options.to_line === 'string' && /\d+/.test(options.to_line)){ - options.to_line = parseInt(options.to_line); - } - if(Number.isInteger(options.to_line)){ - if(options.to_line <= 0){ - throw new Error(`Invalid Option: to_line must be a positive integer greater than 0, got ${JSON.stringify(opts.to_line)}`); - } - }else { - throw new Error(`Invalid Option: to_line must be an integer, got ${JSON.stringify(opts.to_line)}`); - } + throw new Error(`Invalid Option: to must be an integer, got ${JSON.stringify(opts.to)}`); } - this.info = { - bytes: 0, - comment_lines: 0, - empty_lines: 0, - invalid_field_length: 0, - lines: 1, - records: 0 - }; - this.options = options; - this.state = { - bomSkipped: false, - bufBytesStart: 0, - castField: fnCastField, - commenting: false, - // Current error encountered by a record - error: undefined, - enabled: options.from_line === 1, - escaping: false, - escapeIsQuote: isBuffer$1(options.escape) && isBuffer$1(options.quote) && Buffer.compare(options.escape, options.quote) === 0, - // columns can be `false`, `true`, `Array` - expectedRecordLength: Array.isArray(options.columns) ? options.columns.length : undefined, - field: new ResizeableBuffer(20), - firstLineToHeaders: fnFirstLineToHeaders, - needMoreDataSize: Math.max( - // Skip if the remaining buffer smaller than comment - options.comment !== null ? options.comment.length : 0, - // Skip if the remaining buffer can be delimiter - ...options.delimiter.map((delimiter) => delimiter.length), - // Skip if the remaining buffer can be escape sequence - options.quote !== null ? options.quote.length : 0, - ), - previousBuf: undefined, - quoting: false, - stop: false, - rawBuffer: new ResizeableBuffer(100), - record: [], - recordHasError: false, - record_length: 0, - recordDelimiterMaxLength: options.record_delimiter.length === 0 ? 2 : Math.max(...options.record_delimiter.map((v) => v.length)), - trimChars: [Buffer.from(' ', options.encoding)[0], Buffer.from('\t', options.encoding)[0]], - wasQuoting: false, - wasRowDelimiter: false - }; } - // Implementation of `Transform._transform` - _transform(buf, encoding, callback){ - if(this.state.stop === true){ - return; - } - const err = this.__parse(buf, false); - if(err !== undefined){ - this.state.stop = true; - } - callback(err); - } - // Implementation of `Transform._flush` - _flush(callback){ - if(this.state.stop === true){ - return; + // Normalize option `to_line` + if(options.to_line === undefined || options.to_line === null){ + options.to_line = -1; + }else { + if(typeof options.to_line === 'string' && /\d+/.test(options.to_line)){ + options.to_line = parseInt(options.to_line); } - const err = this.__parse(undefined, true); - callback(err); - } - // Central parser implementation - __parse(nextBuf, end){ - const {bom, comment, escape, from_line, ltrim, max_record_size, quote, raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options; - let {record_delimiter} = this.options; - const {bomSkipped, previousBuf, rawBuffer, escapeIsQuote} = this.state; - let buf; - if(previousBuf === undefined){ - if(nextBuf === undefined){ - // Handle empty string - this.push(null); - return; - }else { - buf = nextBuf; + if(Number.isInteger(options.to_line)){ + if(options.to_line <= 0){ + throw new Error(`Invalid Option: to_line must be a positive integer greater than 0, got ${JSON.stringify(opts.to_line)}`); } - }else if(previousBuf !== undefined && nextBuf === undefined){ - buf = previousBuf; }else { - buf = Buffer.concat([previousBuf, nextBuf]); - } - // Handle UTF BOM - if(bomSkipped === false){ - if(bom === false){ - this.state.bomSkipped = true; - }else if(buf.length < 3){ - // No enough data - if(end === false){ - // Wait for more data - this.state.previousBuf = buf; + throw new Error(`Invalid Option: to_line must be an integer, got ${JSON.stringify(opts.to_line)}`); + } + } + return options; +}; + +const isRecordEmpty = function(record){ + return record.every((field) => field == null || field.toString && field.toString().trim() === ''); +}; + +// white space characters +// https://en.wikipedia.org/wiki/Whitespace_character +// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions/Character_Classes#Types +// \f\n\r\t\v\u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff +const tab = 9; +const nl = 10; // \n, 0x0A in hexadecimal, 10 in decimal +const np = 12; +const cr = 13; // \r, 0x0D in hexadécimal, 13 in decimal +const space = 32; +const boms = { + // Note, the following are equals: + // Buffer.from("\ufeff") + // Buffer.from([239, 187, 191]) + // Buffer.from('EFBBBF', 'hex') + 'utf8': Buffer.from([239, 187, 191]), + // Note, the following are equals: + // Buffer.from "\ufeff", 'utf16le + // Buffer.from([255, 254]) + 'utf16le': Buffer.from([255, 254]) +}; + +const transform$1 = function(original_options = {}) { + const info = { + bytes: 0, + comment_lines: 0, + empty_lines: 0, + invalid_field_length: 0, + lines: 1, + records: 0 + }; + const options = normalize_options$1(original_options); + return { + info: info, + original_options: original_options, + options: options, + state: init_state(options), + __needMoreData: function(i, bufLen, end){ + if(end) return false; + const {quote} = this.options; + const {quoting, needMoreDataSize, recordDelimiterMaxLength} = this.state; + const numOfCharLeft = bufLen - i - 1; + const requiredLength = Math.max( + needMoreDataSize, + // Skip if the remaining buffer smaller than record delimiter + recordDelimiterMaxLength, + // Skip if the remaining buffer can be record delimiter following the closing quote + // 1 is for quote.length + quoting ? (quote.length + recordDelimiterMaxLength) : 0, + ); + return numOfCharLeft < requiredLength; + }, + // Central parser implementation + parse: function(nextBuf, end, push, close){ + const {bom, comment, escape, from_line, ltrim, max_record_size, quote, raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options; + let {record_delimiter} = this.options; + const {bomSkipped, previousBuf, rawBuffer, escapeIsQuote} = this.state; + let buf; + if(previousBuf === undefined){ + if(nextBuf === undefined){ + // Handle empty string + close(); return; + }else { + buf = nextBuf; } + }else if(previousBuf !== undefined && nextBuf === undefined){ + buf = previousBuf; }else { - for(const encoding in boms){ - if(boms[encoding].compare(buf, 0, boms[encoding].length) === 0){ - // Skip BOM - const bomLength = boms[encoding].length; - this.state.bufBytesStart += bomLength; - buf = buf.slice(bomLength); - // Renormalize original options with the new encoding - this.__normalizeOptions({...this.__originalOptions, encoding: encoding}); - break; + buf = Buffer.concat([previousBuf, nextBuf]); + } + // Handle UTF BOM + if(bomSkipped === false){ + if(bom === false){ + this.state.bomSkipped = true; + }else if(buf.length < 3){ + // No enough data + if(end === false){ + // Wait for more data + this.state.previousBuf = buf; + return; + } + }else { + for(const encoding in boms){ + if(boms[encoding].compare(buf, 0, boms[encoding].length) === 0){ + // Skip BOM + const bomLength = boms[encoding].length; + this.state.bufBytesStart += bomLength; + buf = buf.slice(bomLength); + // Renormalize original options with the new encoding + this.options = normalize_options$1({...this.original_options, encoding: encoding}); + break; + } } + this.state.bomSkipped = true; } - this.state.bomSkipped = true; } - } - const bufLen = buf.length; - let pos; - for(pos = 0; pos < bufLen; pos++){ - // Ensure we get enough space to look ahead - // There should be a way to move this out of the loop - if(this.__needMoreData(pos, bufLen, end)){ - break; - } - if(this.state.wasRowDelimiter === true){ - this.info.lines++; - this.state.wasRowDelimiter = false; - } - if(to_line !== -1 && this.info.lines > to_line){ - this.state.stop = true; - this.push(null); - return; - } - // Auto discovery of record_delimiter, unix, mac and windows supported - if(this.state.quoting === false && record_delimiter.length === 0){ - const record_delimiterCount = this.__autoDiscoverRecordDelimiter(buf, pos); - if(record_delimiterCount){ - record_delimiter = this.options.record_delimiter; + const bufLen = buf.length; + let pos; + for(pos = 0; pos < bufLen; pos++){ + // Ensure we get enough space to look ahead + // There should be a way to move this out of the loop + if(this.__needMoreData(pos, bufLen, end)){ + break; } - } - const chr = buf[pos]; - if(raw === true){ - rawBuffer.append(chr); - } - if((chr === cr || chr === nl) && this.state.wasRowDelimiter === false){ - this.state.wasRowDelimiter = true; - } - // Previous char was a valid escape char - // treat the current char as a regular char - if(this.state.escaping === true){ - this.state.escaping = false; - }else { - // Escape is only active inside quoted fields - // We are quoting, the char is an escape chr and there is a chr to escape - // if(escape !== null && this.state.quoting === true && chr === escape && pos + 1 < bufLen){ - if(escape !== null && this.state.quoting === true && this.__isEscape(buf, pos, chr) && pos + escape.length < bufLen){ - if(escapeIsQuote){ - if(this.__isQuote(buf, pos+escape.length)){ - this.state.escaping = true; - pos += escape.length - 1; - continue; - } - }else { - this.state.escaping = true; - pos += escape.length - 1; - continue; + if(this.state.wasRowDelimiter === true){ + this.info.lines++; + this.state.wasRowDelimiter = false; + } + if(to_line !== -1 && this.info.lines > to_line){ + this.state.stop = true; + close(); + return; + } + // Auto discovery of record_delimiter, unix, mac and windows supported + if(this.state.quoting === false && record_delimiter.length === 0){ + const record_delimiterCount = this.__autoDiscoverRecordDelimiter(buf, pos); + if(record_delimiterCount){ + record_delimiter = this.options.record_delimiter; } } - // Not currently escaping and chr is a quote - // TODO: need to compare bytes instead of single char - if(this.state.commenting === false && this.__isQuote(buf, pos)){ - if(this.state.quoting === true){ - const nextChr = buf[pos+quote.length]; - const isNextChrTrimable = rtrim && this.__isCharTrimable(nextChr); - const isNextChrComment = comment !== null && this.__compareBytes(comment, buf, pos+quote.length, nextChr); - const isNextChrDelimiter = this.__isDelimiter(buf, pos+quote.length, nextChr); - const isNextChrRecordDelimiter = record_delimiter.length === 0 ? this.__autoDiscoverRecordDelimiter(buf, pos+quote.length) : this.__isRecordDelimiter(nextChr, buf, pos+quote.length); - // Escape a quote - // Treat next char as a regular character - if(escape !== null && this.__isEscape(buf, pos, chr) && this.__isQuote(buf, pos + escape.length)){ + const chr = buf[pos]; + if(raw === true){ + rawBuffer.append(chr); + } + if((chr === cr || chr === nl) && this.state.wasRowDelimiter === false){ + this.state.wasRowDelimiter = true; + } + // Previous char was a valid escape char + // treat the current char as a regular char + if(this.state.escaping === true){ + this.state.escaping = false; + }else { + // Escape is only active inside quoted fields + // We are quoting, the char is an escape chr and there is a chr to escape + // if(escape !== null && this.state.quoting === true && chr === escape && pos + 1 < bufLen){ + if(escape !== null && this.state.quoting === true && this.__isEscape(buf, pos, chr) && pos + escape.length < bufLen){ + if(escapeIsQuote){ + if(this.__isQuote(buf, pos+escape.length)){ + this.state.escaping = true; + pos += escape.length - 1; + continue; + } + }else { + this.state.escaping = true; pos += escape.length - 1; - }else if(!nextChr || isNextChrDelimiter || isNextChrRecordDelimiter || isNextChrComment || isNextChrTrimable){ - this.state.quoting = false; - this.state.wasQuoting = true; - pos += quote.length - 1; continue; - }else if(relax_quotes === false){ - const err = this.__error( - new CsvError$1('CSV_INVALID_CLOSING_QUOTE', [ - 'Invalid Closing Quote:', - `got "${String.fromCharCode(nextChr)}"`, - `at line ${this.info.lines}`, - 'instead of delimiter, record delimiter, trimable character', - '(if activated) or comment', - ], this.options, this.__infoField()) - ); - if(err !== undefined) return err; - }else { - this.state.quoting = false; - this.state.wasQuoting = true; - this.state.field.prepend(quote); - pos += quote.length - 1; } - }else { - if(this.state.field.length !== 0){ - // In relax_quotes mode, treat opening quote preceded by chrs as regular - if(relax_quotes === false){ + } + // Not currently escaping and chr is a quote + // TODO: need to compare bytes instead of single char + if(this.state.commenting === false && this.__isQuote(buf, pos)){ + if(this.state.quoting === true){ + const nextChr = buf[pos+quote.length]; + const isNextChrTrimable = rtrim && this.__isCharTrimable(nextChr); + const isNextChrComment = comment !== null && this.__compareBytes(comment, buf, pos+quote.length, nextChr); + const isNextChrDelimiter = this.__isDelimiter(buf, pos+quote.length, nextChr); + const isNextChrRecordDelimiter = record_delimiter.length === 0 ? this.__autoDiscoverRecordDelimiter(buf, pos+quote.length) : this.__isRecordDelimiter(nextChr, buf, pos+quote.length); + // Escape a quote + // Treat next char as a regular character + if(escape !== null && this.__isEscape(buf, pos, chr) && this.__isQuote(buf, pos + escape.length)){ + pos += escape.length - 1; + }else if(!nextChr || isNextChrDelimiter || isNextChrRecordDelimiter || isNextChrComment || isNextChrTrimable){ + this.state.quoting = false; + this.state.wasQuoting = true; + pos += quote.length - 1; + continue; + }else if(relax_quotes === false){ const err = this.__error( - new CsvError$1('INVALID_OPENING_QUOTE', [ - 'Invalid Opening Quote:', - `a quote is found inside a field at line ${this.info.lines}`, - ], this.options, this.__infoField(), { - field: this.state.field, - }) + new CsvError$1('CSV_INVALID_CLOSING_QUOTE', [ + 'Invalid Closing Quote:', + `got "${String.fromCharCode(nextChr)}"`, + `at line ${this.info.lines}`, + 'instead of delimiter, record delimiter, trimable character', + '(if activated) or comment', + ], this.options, this.__infoField()) ); if(err !== undefined) return err; + }else { + this.state.quoting = false; + this.state.wasQuoting = true; + this.state.field.prepend(quote); + pos += quote.length - 1; } }else { - this.state.quoting = true; - pos += quote.length - 1; - continue; - } - } - } - if(this.state.quoting === false){ - const recordDelimiterLength = this.__isRecordDelimiter(chr, buf, pos); - if(recordDelimiterLength !== 0){ - // Do not emit comments which take a full line - const skipCommentLine = this.state.commenting && (this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0); - if(skipCommentLine){ - this.info.comment_lines++; - // Skip full comment line - }else { - // Activate records emition if above from_line - if(this.state.enabled === false && this.info.lines + (this.state.wasRowDelimiter === true ? 1: 0) >= from_line){ - this.state.enabled = true; - this.__resetField(); - this.__resetRecord(); - pos += recordDelimiterLength - 1; + if(this.state.field.length !== 0){ + // In relax_quotes mode, treat opening quote preceded by chrs as regular + if(relax_quotes === false){ + const err = this.__error( + new CsvError$1('INVALID_OPENING_QUOTE', [ + 'Invalid Opening Quote:', + `a quote is found inside a field at line ${this.info.lines}`, + ], this.options, this.__infoField(), { + field: this.state.field, + }) + ); + if(err !== undefined) return err; + } + }else { + this.state.quoting = true; + pos += quote.length - 1; continue; } - // Skip if line is empty and skip_empty_lines activated - if(skip_empty_lines === true && this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0){ - this.info.empty_lines++; - pos += recordDelimiterLength - 1; - continue; + } + } + if(this.state.quoting === false){ + const recordDelimiterLength = this.__isRecordDelimiter(chr, buf, pos); + if(recordDelimiterLength !== 0){ + // Do not emit comments which take a full line + const skipCommentLine = this.state.commenting && (this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0); + if(skipCommentLine){ + this.info.comment_lines++; + // Skip full comment line + }else { + // Activate records emition if above from_line + if(this.state.enabled === false && this.info.lines + (this.state.wasRowDelimiter === true ? 1: 0) >= from_line){ + this.state.enabled = true; + this.__resetField(); + this.__resetRecord(); + pos += recordDelimiterLength - 1; + continue; + } + // Skip if line is empty and skip_empty_lines activated + if(skip_empty_lines === true && this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0){ + this.info.empty_lines++; + pos += recordDelimiterLength - 1; + continue; + } + this.info.bytes = this.state.bufBytesStart + pos; + const errField = this.__onField(); + if(errField !== undefined) return errField; + this.info.bytes = this.state.bufBytesStart + pos + recordDelimiterLength; + const errRecord = this.__onRecord(push); + if(errRecord !== undefined) return errRecord; + if(to !== -1 && this.info.records >= to){ + this.state.stop = true; + close(); + return; + } } + this.state.commenting = false; + pos += recordDelimiterLength - 1; + continue; + } + if(this.state.commenting){ + continue; + } + const commentCount = comment === null ? 0 : this.__compareBytes(comment, buf, pos, chr); + if(commentCount !== 0){ + this.state.commenting = true; + continue; + } + const delimiterLength = this.__isDelimiter(buf, pos, chr); + if(delimiterLength !== 0){ this.info.bytes = this.state.bufBytesStart + pos; const errField = this.__onField(); if(errField !== undefined) return errField; - this.info.bytes = this.state.bufBytesStart + pos + recordDelimiterLength; - const errRecord = this.__onRecord(); - if(errRecord !== undefined) return errRecord; - if(to !== -1 && this.info.records >= to){ - this.state.stop = true; - this.push(null); - return; - } + pos += delimiterLength - 1; + continue; } - this.state.commenting = false; - pos += recordDelimiterLength - 1; - continue; } - if(this.state.commenting){ - continue; - } - const commentCount = comment === null ? 0 : this.__compareBytes(comment, buf, pos, chr); - if(commentCount !== 0){ - this.state.commenting = true; - continue; - } - const delimiterLength = this.__isDelimiter(buf, pos, chr); - if(delimiterLength !== 0){ - this.info.bytes = this.state.bufBytesStart + pos; - const errField = this.__onField(); - if(errField !== undefined) return errField; - pos += delimiterLength - 1; - continue; + } + if(this.state.commenting === false){ + if(max_record_size !== 0 && this.state.record_length + this.state.field.length > max_record_size){ + const err = this.__error( + new CsvError$1('CSV_MAX_RECORD_SIZE', [ + 'Max Record Size:', + 'record exceed the maximum number of tolerated bytes', + `of ${max_record_size}`, + `at line ${this.info.lines}`, + ], this.options, this.__infoField()) + ); + if(err !== undefined) return err; } } - } - if(this.state.commenting === false){ - if(max_record_size !== 0 && this.state.record_length + this.state.field.length > max_record_size){ + const lappend = ltrim === false || this.state.quoting === true || this.state.field.length !== 0 || !this.__isCharTrimable(chr); + // rtrim in non quoting is handle in __onField + const rappend = rtrim === false || this.state.wasQuoting === false; + if(lappend === true && rappend === true){ + this.state.field.append(chr); + }else if(rtrim === true && !this.__isCharTrimable(chr)){ const err = this.__error( - new CsvError$1('CSV_MAX_RECORD_SIZE', [ - 'Max Record Size:', - 'record exceed the maximum number of tolerated bytes', - `of ${max_record_size}`, + new CsvError$1('CSV_NON_TRIMABLE_CHAR_AFTER_CLOSING_QUOTE', [ + 'Invalid Closing Quote:', + 'found non trimable byte after quote', `at line ${this.info.lines}`, ], this.options, this.__infoField()) ); if(err !== undefined) return err; } } - const lappend = ltrim === false || this.state.quoting === true || this.state.field.length !== 0 || !this.__isCharTrimable(chr); - // rtrim in non quoting is handle in __onField - const rappend = rtrim === false || this.state.wasQuoting === false; - if(lappend === true && rappend === true){ - this.state.field.append(chr); - }else if(rtrim === true && !this.__isCharTrimable(chr)){ - const err = this.__error( - new CsvError$1('CSV_NON_TRIMABLE_CHAR_AFTER_CLOSING_QUOTE', [ - 'Invalid Closing Quote:', - 'found non trimable byte after quote', - `at line ${this.info.lines}`, - ], this.options, this.__infoField()) - ); - if(err !== undefined) return err; - } - } - if(end === true){ - // Ensure we are not ending in a quoting state - if(this.state.quoting === true){ - const err = this.__error( - new CsvError$1('CSV_QUOTE_NOT_CLOSED', [ - 'Quote Not Closed:', - `the parsing is finished with an opening quote at line ${this.info.lines}`, - ], this.options, this.__infoField()) - ); - if(err !== undefined) return err; + if(end === true){ + // Ensure we are not ending in a quoting state + if(this.state.quoting === true){ + const err = this.__error( + new CsvError$1('CSV_QUOTE_NOT_CLOSED', [ + 'Quote Not Closed:', + `the parsing is finished with an opening quote at line ${this.info.lines}`, + ], this.options, this.__infoField()) + ); + if(err !== undefined) return err; + }else { + // Skip last line if it has no characters + if(this.state.wasQuoting === true || this.state.record.length !== 0 || this.state.field.length !== 0){ + this.info.bytes = this.state.bufBytesStart + pos; + const errField = this.__onField(); + if(errField !== undefined) return errField; + const errRecord = this.__onRecord(push); + if(errRecord !== undefined) return errRecord; + }else if(this.state.wasRowDelimiter === true){ + this.info.empty_lines++; + }else if(this.state.commenting === true){ + this.info.comment_lines++; + } + } }else { - // Skip last line if it has no characters - if(this.state.wasQuoting === true || this.state.record.length !== 0 || this.state.field.length !== 0){ - this.info.bytes = this.state.bufBytesStart + pos; - const errField = this.__onField(); - if(errField !== undefined) return errField; - const errRecord = this.__onRecord(); - if(errRecord !== undefined) return errRecord; - }else if(this.state.wasRowDelimiter === true){ - this.info.empty_lines++; - }else if(this.state.commenting === true){ - this.info.comment_lines++; + this.state.bufBytesStart += pos; + this.state.previousBuf = buf.slice(pos); + } + if(this.state.wasRowDelimiter === true){ + this.info.lines++; + this.state.wasRowDelimiter = false; + } + }, + __onRecord: function(push){ + const {columns, group_columns_by_name, encoding, info, from, relax_column_count, relax_column_count_less, relax_column_count_more, raw, skip_records_with_empty_values} = this.options; + const {enabled, record} = this.state; + if(enabled === false){ + return this.__resetRecord(); + } + // Convert the first line into column names + const recordLength = record.length; + if(columns === true){ + if(skip_records_with_empty_values === true && isRecordEmpty(record)){ + this.__resetRecord(); + return; + } + return this.__firstLineToColumns(record); + } + if(columns === false && this.info.records === 0){ + this.state.expectedRecordLength = recordLength; + } + if(recordLength !== this.state.expectedRecordLength){ + const err = columns === false ? + new CsvError$1('CSV_RECORD_INCONSISTENT_FIELDS_LENGTH', [ + 'Invalid Record Length:', + `expect ${this.state.expectedRecordLength},`, + `got ${recordLength} on line ${this.info.lines}`, + ], this.options, this.__infoField(), { + record: record, + }) + : + new CsvError$1('CSV_RECORD_INCONSISTENT_COLUMNS', [ + 'Invalid Record Length:', + `columns length is ${columns.length},`, // rename columns + `got ${recordLength} on line ${this.info.lines}`, + ], this.options, this.__infoField(), { + record: record, + }); + if(relax_column_count === true || + (relax_column_count_less === true && recordLength < this.state.expectedRecordLength) || + (relax_column_count_more === true && recordLength > this.state.expectedRecordLength)){ + this.info.invalid_field_length++; + this.state.error = err; + // Error is undefined with skip_records_with_error + }else { + const finalErr = this.__error(err); + if(finalErr) return finalErr; } } - }else { - this.state.bufBytesStart += pos; - this.state.previousBuf = buf.slice(pos); - } - if(this.state.wasRowDelimiter === true){ - this.info.lines++; - this.state.wasRowDelimiter = false; - } - } - __onRecord(){ - const {columns, group_columns_by_name, encoding, info, from, relax_column_count, relax_column_count_less, relax_column_count_more, raw, skip_records_with_empty_values} = this.options; - const {enabled, record} = this.state; - if(enabled === false){ - return this.__resetRecord(); - } - // Convert the first line into column names - const recordLength = record.length; - if(columns === true){ if(skip_records_with_empty_values === true && isRecordEmpty(record)){ this.__resetRecord(); return; } - return this.__firstLineToColumns(record); - } - if(columns === false && this.info.records === 0){ - this.state.expectedRecordLength = recordLength; - } - if(recordLength !== this.state.expectedRecordLength){ - const err = columns === false ? - new CsvError$1('CSV_RECORD_INCONSISTENT_FIELDS_LENGTH', [ - 'Invalid Record Length:', - `expect ${this.state.expectedRecordLength},`, - `got ${recordLength} on line ${this.info.lines}`, - ], this.options, this.__infoField(), { - record: record, - }) - : - new CsvError$1('CSV_RECORD_INCONSISTENT_COLUMNS', [ - 'Invalid Record Length:', - `columns length is ${columns.length},`, // rename columns - `got ${recordLength} on line ${this.info.lines}`, - ], this.options, this.__infoField(), { - record: record, - }); - if(relax_column_count === true || - (relax_column_count_less === true && recordLength < this.state.expectedRecordLength) || - (relax_column_count_more === true && recordLength > this.state.expectedRecordLength)){ - this.info.invalid_field_length++; - this.state.error = err; - // Error is undefined with skip_records_with_error - }else { - const finalErr = this.__error(err); - if(finalErr) return finalErr; + if(this.state.recordHasError === true){ + this.__resetRecord(); + this.state.recordHasError = false; + return; } - } - if(skip_records_with_empty_values === true && isRecordEmpty(record)){ - this.__resetRecord(); - return; - } - if(this.state.recordHasError === true){ - this.__resetRecord(); - this.state.recordHasError = false; - return; - } - this.info.records++; - if(from === 1 || this.info.records >= from){ - const {objname} = this.options; - // With columns, records are object - if(columns !== false){ - const obj = {}; - // Transform record array to an object - for(let i = 0, l = record.length; i < l; i++){ - if(columns[i] === undefined || columns[i].disabled) continue; - // Turn duplicate columns into an array - if (group_columns_by_name === true && obj[columns[i].name] !== undefined) { - if (Array.isArray(obj[columns[i].name])) { - obj[columns[i].name] = obj[columns[i].name].concat(record[i]); + this.info.records++; + if(from === 1 || this.info.records >= from){ + const {objname} = this.options; + // With columns, records are object + if(columns !== false){ + const obj = {}; + // Transform record array to an object + for(let i = 0, l = record.length; i < l; i++){ + if(columns[i] === undefined || columns[i].disabled) continue; + // Turn duplicate columns into an array + if (group_columns_by_name === true && obj[columns[i].name] !== undefined) { + if (Array.isArray(obj[columns[i].name])) { + obj[columns[i].name] = obj[columns[i].name].concat(record[i]); + } else { + obj[columns[i].name] = [obj[columns[i].name], record[i]]; + } } else { - obj[columns[i].name] = [obj[columns[i].name], record[i]]; + obj[columns[i].name] = record[i]; } - } else { - obj[columns[i].name] = record[i]; } - } - // Without objname (default) - if(raw === true || info === true){ - const extRecord = Object.assign( - {record: obj}, - (raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}), - (info === true ? {info: this.__infoRecord()}: {}) - ); - const err = this.__push( - objname === undefined ? extRecord : [obj[objname], extRecord] - ); - if(err){ - return err; - } - }else { - const err = this.__push( - objname === undefined ? obj : [obj[objname], obj] - ); - if(err){ - return err; - } - } - // Without columns, records are array - }else { - if(raw === true || info === true){ - const extRecord = Object.assign( - {record: record}, - raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}, - info === true ? {info: this.__infoRecord()}: {} - ); - const err = this.__push( - objname === undefined ? extRecord : [record[objname], extRecord] - ); - if(err){ - return err; + // Without objname (default) + if(raw === true || info === true){ + const extRecord = Object.assign( + {record: obj}, + (raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}), + (info === true ? {info: this.__infoRecord()}: {}) + ); + const err = this.__push( + objname === undefined ? extRecord : [obj[objname], extRecord] + , push); + if(err){ + return err; + } + }else { + const err = this.__push( + objname === undefined ? obj : [obj[objname], obj] + , push); + if(err){ + return err; + } } + // Without columns, records are array }else { - const err = this.__push( - objname === undefined ? record : [record[objname], record] - ); - if(err){ - return err; + if(raw === true || info === true){ + const extRecord = Object.assign( + {record: record}, + raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}, + info === true ? {info: this.__infoRecord()}: {} + ); + const err = this.__push( + objname === undefined ? extRecord : [record[objname], extRecord] + , push); + if(err){ + return err; + } + }else { + const err = this.__push( + objname === undefined ? record : [record[objname], record] + , push); + if(err){ + return err; + } } } } - } - this.__resetRecord(); - } - __firstLineToColumns(record){ - const {firstLineToHeaders} = this.state; - try{ - const headers = firstLineToHeaders === undefined ? record : firstLineToHeaders.call(null, record); - if(!Array.isArray(headers)){ - return this.__error( - new CsvError$1('CSV_INVALID_COLUMN_MAPPING', [ - 'Invalid Column Mapping:', - 'expect an array from column function,', - `got ${JSON.stringify(headers)}` - ], this.options, this.__infoField(), { - headers: headers, - }) - ); - } - const normalizedHeaders = normalizeColumnsArray(headers); - this.state.expectedRecordLength = normalizedHeaders.length; - this.options.columns = normalizedHeaders; this.__resetRecord(); - return; - }catch(err){ - return err; - } - } - __resetRecord(){ - if(this.options.raw === true){ - this.state.rawBuffer.reset(); - } - this.state.error = undefined; - this.state.record = []; - this.state.record_length = 0; - } - __onField(){ - const {cast, encoding, rtrim, max_record_size} = this.options; - const {enabled, wasQuoting} = this.state; - // Short circuit for the from_line options - if(enabled === false){ - return this.__resetField(); - } - let field = this.state.field.toString(encoding); - if(rtrim === true && wasQuoting === false){ - field = field.trimRight(); - } - if(cast === true){ - const [err, f] = this.__cast(field); - if(err !== undefined) return err; - field = f; - } - this.state.record.push(field); - // Increment record length if record size must not exceed a limit - if(max_record_size !== 0 && typeof field === 'string'){ - this.state.record_length += field.length; - } - this.__resetField(); - } - __resetField(){ - this.state.field.reset(); - this.state.wasQuoting = false; - } - __push(record){ - const {on_record} = this.options; - if(on_record !== undefined){ - const info = this.__infoRecord(); + }, + __firstLineToColumns: function(record){ + const {firstLineToHeaders} = this.state; try{ - record = on_record.call(null, record, info); + const headers = firstLineToHeaders === undefined ? record : firstLineToHeaders.call(null, record); + if(!Array.isArray(headers)){ + return this.__error( + new CsvError$1('CSV_INVALID_COLUMN_MAPPING', [ + 'Invalid Column Mapping:', + 'expect an array from column function,', + `got ${JSON.stringify(headers)}` + ], this.options, this.__infoField(), { + headers: headers, + }) + ); + } + const normalizedHeaders = normalize_columns_array(headers); + this.state.expectedRecordLength = normalizedHeaders.length; + this.options.columns = normalizedHeaders; + this.__resetRecord(); + return; }catch(err){ return err; } - if(record === undefined || record === null){ return; } - } - this.push(record); - } - // Return a tuple with the error and the casted value - __cast(field){ - const {columns, relax_column_count} = this.options; - const isColumns = Array.isArray(columns); - // Dont loose time calling cast - // because the final record is an object - // and this field can't be associated to a key present in columns - if(isColumns === true && relax_column_count && this.options.columns.length <= this.state.record.length){ - return [undefined, undefined]; - } - if(this.state.castField !== null){ - try{ + }, + __resetRecord: function(){ + if(this.options.raw === true){ + this.state.rawBuffer.reset(); + } + this.state.error = undefined; + this.state.record = []; + this.state.record_length = 0; + }, + __onField: function(){ + const {cast, encoding, rtrim, max_record_size} = this.options; + const {enabled, wasQuoting} = this.state; + // Short circuit for the from_line options + if(enabled === false){ + return this.__resetField(); + } + let field = this.state.field.toString(encoding); + if(rtrim === true && wasQuoting === false){ + field = field.trimRight(); + } + if(cast === true){ + const [err, f] = this.__cast(field); + if(err !== undefined) return err; + field = f; + } + this.state.record.push(field); + // Increment record length if record size must not exceed a limit + if(max_record_size !== 0 && typeof field === 'string'){ + this.state.record_length += field.length; + } + this.__resetField(); + }, + __resetField: function(){ + this.state.field.reset(); + this.state.wasQuoting = false; + }, + __push: function(record, push){ + const {on_record} = this.options; + if(on_record !== undefined){ + const info = this.__infoRecord(); + try{ + record = on_record.call(null, record, info); + }catch(err){ + return err; + } + if(record === undefined || record === null){ return; } + } + push(record); + }, + // Return a tuple with the error and the casted value + __cast: function(field){ + const {columns, relax_column_count} = this.options; + const isColumns = Array.isArray(columns); + // Dont loose time calling cast + // because the final record is an object + // and this field can't be associated to a key present in columns + if(isColumns === true && relax_column_count && this.options.columns.length <= this.state.record.length){ + return [undefined, undefined]; + } + if(this.state.castField !== null){ + try{ + const info = this.__infoField(); + return [undefined, this.state.castField.call(null, field, info)]; + }catch(err){ + return [err]; + } + } + if(this.__isFloat(field)){ + return [undefined, parseFloat(field)]; + }else if(this.options.cast_date !== false){ const info = this.__infoField(); - return [undefined, this.state.castField.call(null, field, info)]; - }catch(err){ - return [err]; + return [undefined, this.options.cast_date.call(null, field, info)]; } - } - if(this.__isFloat(field)){ - return [undefined, parseFloat(field)]; - }else if(this.options.cast_date !== false){ - const info = this.__infoField(); - return [undefined, this.options.cast_date.call(null, field, info)]; - } - return [undefined, field]; - } - // Helper to test if a character is a space or a line delimiter - __isCharTrimable(chr){ - return chr === space || chr === tab || chr === cr || chr === nl || chr === np; - } - // Keep it in case we implement the `cast_int` option - // __isInt(value){ - // // return Number.isInteger(parseInt(value)) - // // return !isNaN( parseInt( obj ) ); - // return /^(\-|\+)?[1-9][0-9]*$/.test(value) - // } - __isFloat(value){ - return (value - parseFloat(value) + 1) >= 0; // Borrowed from jquery - } - __compareBytes(sourceBuf, targetBuf, targetPos, firstByte){ - if(sourceBuf[0] !== firstByte) return 0; - const sourceLength = sourceBuf.length; - for(let i = 1; i < sourceLength; i++){ - if(sourceBuf[i] !== targetBuf[targetPos+i]) return 0; - } - return sourceLength; - } - __needMoreData(i, bufLen, end){ - if(end) return false; - const {quote} = this.options; - const {quoting, needMoreDataSize, recordDelimiterMaxLength} = this.state; - const numOfCharLeft = bufLen - i - 1; - const requiredLength = Math.max( - needMoreDataSize, - // Skip if the remaining buffer smaller than record delimiter - recordDelimiterMaxLength, - // Skip if the remaining buffer can be record delimiter following the closing quote - // 1 is for quote.length - quoting ? (quote.length + recordDelimiterMaxLength) : 0, - ); - return numOfCharLeft < requiredLength; - } - __isDelimiter(buf, pos, chr){ - const {delimiter, ignore_last_delimiters} = this.options; - if(ignore_last_delimiters === true && this.state.record.length === this.options.columns.length - 1){ - return 0; - }else if(ignore_last_delimiters !== false && typeof ignore_last_delimiters === 'number' && this.state.record.length === ignore_last_delimiters - 1){ - return 0; - } - loop1: for(let i = 0; i < delimiter.length; i++){ - const del = delimiter[i]; - if(del[0] === chr){ - for(let j = 1; j < del.length; j++){ - if(del[j] !== buf[pos+j]) continue loop1; + return [undefined, field]; + }, + // Helper to test if a character is a space or a line delimiter + __isCharTrimable: function(chr){ + return chr === space || chr === tab || chr === cr || chr === nl || chr === np; + }, + // Keep it in case we implement the `cast_int` option + // __isInt(value){ + // // return Number.isInteger(parseInt(value)) + // // return !isNaN( parseInt( obj ) ); + // return /^(\-|\+)?[1-9][0-9]*$/.test(value) + // } + __isFloat: function(value){ + return (value - parseFloat(value) + 1) >= 0; // Borrowed from jquery + }, + __compareBytes: function(sourceBuf, targetBuf, targetPos, firstByte){ + if(sourceBuf[0] !== firstByte) return 0; + const sourceLength = sourceBuf.length; + for(let i = 1; i < sourceLength; i++){ + if(sourceBuf[i] !== targetBuf[targetPos+i]) return 0; + } + return sourceLength; + }, + __isDelimiter: function(buf, pos, chr){ + const {delimiter, ignore_last_delimiters} = this.options; + if(ignore_last_delimiters === true && this.state.record.length === this.options.columns.length - 1){ + return 0; + }else if(ignore_last_delimiters !== false && typeof ignore_last_delimiters === 'number' && this.state.record.length === ignore_last_delimiters - 1){ + return 0; + } + loop1: for(let i = 0; i < delimiter.length; i++){ + const del = delimiter[i]; + if(del[0] === chr){ + for(let j = 1; j < del.length; j++){ + if(del[j] !== buf[pos+j]) continue loop1; + } + return del.length; } - return del.length; } - } - return 0; - } - __isRecordDelimiter(chr, buf, pos){ - const {record_delimiter} = this.options; - const recordDelimiterLength = record_delimiter.length; - loop1: for(let i = 0; i < recordDelimiterLength; i++){ - const rd = record_delimiter[i]; - const rdLength = rd.length; - if(rd[0] !== chr){ - continue; + return 0; + }, + __isRecordDelimiter: function(chr, buf, pos){ + const {record_delimiter} = this.options; + const recordDelimiterLength = record_delimiter.length; + loop1: for(let i = 0; i < recordDelimiterLength; i++){ + const rd = record_delimiter[i]; + const rdLength = rd.length; + if(rd[0] !== chr){ + continue; + } + for(let j = 1; j < rdLength; j++){ + if(rd[j] !== buf[pos+j]){ + continue loop1; + } + } + return rd.length; } - for(let j = 1; j < rdLength; j++){ - if(rd[j] !== buf[pos+j]){ - continue loop1; + return 0; + }, + __isEscape: function(buf, pos, chr){ + const {escape} = this.options; + if(escape === null) return false; + const l = escape.length; + if(escape[0] === chr){ + for(let i = 0; i < l; i++){ + if(escape[i] !== buf[pos+i]){ + return false; + } } + return true; } - return rd.length; - } - return 0; - } - __isEscape(buf, pos, chr){ - const {escape} = this.options; - if(escape === null) return false; - const l = escape.length; - if(escape[0] === chr){ + return false; + }, + __isQuote: function(buf, pos){ + const {quote} = this.options; + if(quote === null) return false; + const l = quote.length; for(let i = 0; i < l; i++){ - if(escape[i] !== buf[pos+i]){ + if(quote[i] !== buf[pos+i]){ return false; } } return true; - } - return false; - } - __isQuote(buf, pos){ - const {quote} = this.options; - if(quote === null) return false; - const l = quote.length; - for(let i = 0; i < l; i++){ - if(quote[i] !== buf[pos+i]){ - return false; - } - } - return true; - } - __autoDiscoverRecordDelimiter(buf, pos){ - const {encoding} = this.options; - const chr = buf[pos]; - if(chr === cr){ - if(buf[pos+1] === nl){ - this.options.record_delimiter.push(Buffer.from('\r\n', encoding)); - this.state.recordDelimiterMaxLength = 2; - return 2; - }else { - this.options.record_delimiter.push(Buffer.from('\r', encoding)); + }, + __autoDiscoverRecordDelimiter: function(buf, pos){ + const {encoding} = this.options; + const chr = buf[pos]; + if(chr === cr){ + if(buf[pos+1] === nl){ + this.options.record_delimiter.push(Buffer.from('\r\n', encoding)); + this.state.recordDelimiterMaxLength = 2; + return 2; + }else { + this.options.record_delimiter.push(Buffer.from('\r', encoding)); + this.state.recordDelimiterMaxLength = 1; + return 1; + } + }else if(chr === nl){ + this.options.record_delimiter.push(Buffer.from('\n', encoding)); this.state.recordDelimiterMaxLength = 1; return 1; } - }else if(chr === nl){ - this.options.record_delimiter.push(Buffer.from('\n', encoding)); - this.state.recordDelimiterMaxLength = 1; - return 1; - } - return 0; - } - __error(msg){ - const {encoding, raw, skip_records_with_error} = this.options; - const err = typeof msg === 'string' ? new Error(msg) : msg; - if(skip_records_with_error){ - this.state.recordHasError = true; - this.emit('skip', err, raw ? this.state.rawBuffer.toString(encoding) : undefined); - return undefined; - }else { - return err; + return 0; + }, + __error: function(msg){ + const {encoding, raw, skip_records_with_error} = this.options; + const err = typeof msg === 'string' ? new Error(msg) : msg; + if(skip_records_with_error){ + this.state.recordHasError = true; + if(this.options.on_skip !== undefined){ + this.options.on_skip(err, raw ? this.state.rawBuffer.toString(encoding) : undefined); + } + // this.emit('skip', err, raw ? this.state.rawBuffer.toString(encoding) : undefined); + return undefined; + }else { + return err; + } + }, + __infoDataSet: function(){ + return { + ...this.info, + columns: this.options.columns + }; + }, + __infoRecord: function(){ + const {columns, raw, encoding} = this.options; + return { + ...this.__infoDataSet(), + error: this.state.error, + header: columns === true, + index: this.state.record.length, + raw: raw ? this.state.rawBuffer.toString(encoding) : undefined + }; + }, + __infoField: function(){ + const {columns} = this.options; + const isColumns = Array.isArray(columns); + return { + ...this.__infoRecord(), + column: isColumns === true ? + (columns.length > this.state.record.length ? + columns[this.state.record.length].name : + null + ) : + this.state.record.length, + quoting: this.state.wasQuoting, + }; } - } - __infoDataSet(){ - return { - ...this.info, - columns: this.options.columns + }; +}; + +class Parser extends Transform { + constructor(opts = {}){ + super({...{readableObjectMode: true}, ...opts, encoding: null}); + this.api = transform$1(opts); + this.api.options.on_skip = (err, chunk) => { + this.emit('skip', err, chunk); }; + // Backward compatibility + this.state = this.api.state; + this.options = this.api.options; + this.info = this.api.info; } - __infoRecord(){ - const {columns, raw, encoding} = this.options; - return { - ...this.__infoDataSet(), - error: this.state.error, - header: columns === true, - index: this.state.record.length, - raw: raw ? this.state.rawBuffer.toString(encoding) : undefined - }; + // Implementation of `Transform._transform` + _transform(buf, encoding, callback){ + if(this.state.stop === true){ + return; + } + const err = this.api.parse(buf, false, (record) => { + this.push.call(this, record); + }, () => { + this.push.call(this, null); + }); + if(err !== undefined){ + this.state.stop = true; + } + callback(err); } - __infoField(){ - const {columns} = this.options; - const isColumns = Array.isArray(columns); - return { - ...this.__infoRecord(), - column: isColumns === true ? - (columns.length > this.state.record.length ? - columns[this.state.record.length].name : - null - ) : - this.state.record.length, - quoting: this.state.wasQuoting, - }; + // Implementation of `Transform._flush` + _flush(callback){ + if(this.state.stop === true){ + return; + } + const err = this.api.parse(undefined, true, (record) => { + this.push.call(this, record); + }, () => { + this.push.call(this, null); + }); + callback(err); } } @@ -6531,7 +6595,7 @@ const parse = function(){ const type = typeof argument; if(data === undefined && (typeof argument === 'string' || isBuffer$1(argument))){ data = argument; - }else if(options === undefined && isObject$1(argument)){ + }else if(options === undefined && is_object$1(argument)){ options = argument; }else if(callback === undefined && type === 'function'){ callback = argument; @@ -6556,10 +6620,10 @@ const parse = function(){ } }); parser.on('error', function(err){ - callback(err, undefined, parser.__infoDataSet()); + callback(err, undefined, parser.api.__infoDataSet()); }); parser.on('end', function(){ - callback(undefined, records, parser.__infoDataSet()); + callback(undefined, records, parser.api.__infoDataSet()); }); } if(data !== undefined){ @@ -6722,8 +6786,6 @@ const transform = function(){ return transformer; }; -const bom_utf8 = Buffer.from([239, 187, 191]); - class CsvError extends Error { constructor(code, message, ...contexts) { if(Array.isArray(message)) message = message.join(' '); @@ -6741,16 +6803,10 @@ class CsvError extends Error { } } -const isObject = function(obj){ +const is_object = function(obj){ return typeof obj === 'object' && obj !== null && ! Array.isArray(obj); }; -const underscore = function(str){ - return str.replace(/([A-Z])/g, function(_, match){ - return '_' + match.toLowerCase(); - }); -}; - // Lodash implementation of `get` const charCodeOfDot = '.'.charCodeAt(0); @@ -6828,441 +6884,472 @@ const get = function(object, path){ return (index && index === length) ? object : undefined; }; -class Stringifier extends Transform { - constructor(opts = {}){ - super({...{writableObjectMode: true}, ...opts}); - const options = {}; - let err; - // Merge with user options - for(const opt in opts){ - options[underscore(opt)] = opts[opt]; - } - if((err = this.normalize(options)) !== undefined) throw err; - switch(options.record_delimiter){ - case 'auto': - options.record_delimiter = null; - break; - case 'unix': - options.record_delimiter = "\n"; - break; - case 'mac': - options.record_delimiter = "\r"; - break; - case 'windows': - options.record_delimiter = "\r\n"; - break; - case 'ascii': - options.record_delimiter = "\u001e"; - break; - case 'unicode': - options.record_delimiter = "\u2028"; - break; - } - // Expose options - this.options = options; - // Internal state - this.state = { - stop: false - }; - // Information - this.info = { - records: 0 - }; +const normalize_columns = function(columns){ + if(columns === undefined || columns === null){ + return [undefined, undefined]; } - normalize(options){ - // Normalize option `bom` - if(options.bom === undefined || options.bom === null || options.bom === false){ - options.bom = false; - }else if(options.bom !== true){ - return new CsvError('CSV_OPTION_BOOLEAN_INVALID_TYPE', [ - 'option `bom` is optional and must be a boolean value,', - `got ${JSON.stringify(options.bom)}` - ]); - } - // Normalize option `delimiter` - if(options.delimiter === undefined || options.delimiter === null){ - options.delimiter = ','; - }else if(isBuffer$1(options.delimiter)){ - options.delimiter = options.delimiter.toString(); - }else if(typeof options.delimiter !== 'string'){ - return new CsvError('CSV_OPTION_DELIMITER_INVALID_TYPE', [ - 'option `delimiter` must be a buffer or a string,', - `got ${JSON.stringify(options.delimiter)}` - ]); - } - // Normalize option `quote` - if(options.quote === undefined || options.quote === null){ - options.quote = '"'; - }else if(options.quote === true){ - options.quote = '"'; - }else if(options.quote === false){ - options.quote = ''; - }else if (isBuffer$1(options.quote)){ - options.quote = options.quote.toString(); - }else if(typeof options.quote !== 'string'){ - return new CsvError('CSV_OPTION_QUOTE_INVALID_TYPE', [ - 'option `quote` must be a boolean, a buffer or a string,', - `got ${JSON.stringify(options.quote)}` - ]); + if(typeof columns !== 'object'){ + return [Error('Invalid option "columns": expect an array or an object')]; + } + if(!Array.isArray(columns)){ + const newcolumns = []; + for(const k in columns){ + newcolumns.push({ + key: k, + header: columns[k] + }); } - // Normalize option `quoted` - if(options.quoted === undefined || options.quoted === null){ - options.quoted = false; - } - // Normalize option `quoted_empty` - if(options.quoted_empty === undefined || options.quoted_empty === null){ - options.quoted_empty = undefined; - } - // Normalize option `quoted_match` - if(options.quoted_match === undefined || options.quoted_match === null || options.quoted_match === false){ - options.quoted_match = null; - }else if(!Array.isArray(options.quoted_match)){ - options.quoted_match = [options.quoted_match]; - } - if(options.quoted_match){ - for(const quoted_match of options.quoted_match){ - const isString = typeof quoted_match === 'string'; - const isRegExp = quoted_match instanceof RegExp; - if(!isString && !isRegExp){ - return Error(`Invalid Option: quoted_match must be a string or a regex, got ${JSON.stringify(quoted_match)}`); + columns = newcolumns; + }else { + const newcolumns = []; + for(const column of columns){ + if(typeof column === 'string'){ + newcolumns.push({ + key: column, + header: column + }); + }else if(typeof column === 'object' && column !== null && !Array.isArray(column)){ + if(!column.key){ + return [Error('Invalid column definition: property "key" is required')]; } + if(column.header === undefined){ + column.header = column.key; + } + newcolumns.push(column); + }else { + return [Error('Invalid column definition: expect a string or an object')]; } } - // Normalize option `quoted_string` - if(options.quoted_string === undefined || options.quoted_string === null){ - options.quoted_string = false; - } - // Normalize option `eof` - if(options.eof === undefined || options.eof === null){ - options.eof = true; - } - // Normalize option `escape` - if(options.escape === undefined || options.escape === null){ - options.escape = '"'; - }else if(isBuffer$1(options.escape)){ - options.escape = options.escape.toString(); - }else if(typeof options.escape !== 'string'){ - return Error(`Invalid Option: escape must be a buffer or a string, got ${JSON.stringify(options.escape)}`); - } - if (options.escape.length > 1){ - return Error(`Invalid Option: escape must be one character, got ${options.escape.length} characters`); - } - // Normalize option `header` - if(options.header === undefined || options.header === null){ - options.header = false; - } - // Normalize option `columns` - const [err, columns] = this.normalize_columns(options.columns); - if(err) return err; - options.columns = columns; - // Normalize option `quoted` - if(options.quoted === undefined || options.quoted === null){ - options.quoted = false; - } - // Normalize option `cast` - if(options.cast === undefined || options.cast === null){ - options.cast = {}; - } - // Normalize option cast.bigint - if(options.cast.bigint === undefined || options.cast.bigint === null){ - // Cast boolean to string by default - options.cast.bigint = value => '' + value; - } - // Normalize option cast.boolean - if(options.cast.boolean === undefined || options.cast.boolean === null){ - // Cast boolean to string by default - options.cast.boolean = value => value ? '1' : ''; - } - // Normalize option cast.date - if(options.cast.date === undefined || options.cast.date === null){ - // Cast date to timestamp string by default - options.cast.date = value => '' + value.getTime(); - } - // Normalize option cast.number - if(options.cast.number === undefined || options.cast.number === null){ - // Cast number to string using native casting by default - options.cast.number = value => '' + value; - } - // Normalize option cast.object - if(options.cast.object === undefined || options.cast.object === null){ - // Stringify object as JSON by default - options.cast.object = value => JSON.stringify(value); - } - // Normalize option cast.string - if(options.cast.string === undefined || options.cast.string === null){ - // Leave string untouched - options.cast.string = function(value){return value;}; - } - // Normalize option `record_delimiter` - if(options.record_delimiter === undefined || options.record_delimiter === null){ - options.record_delimiter = '\n'; - }else if(isBuffer$1(options.record_delimiter)){ - options.record_delimiter = options.record_delimiter.toString(); - }else if(typeof options.record_delimiter !== 'string'){ - return Error(`Invalid Option: record_delimiter must be a buffer or a string, got ${JSON.stringify(options.record_delimiter)}`); - } + columns = newcolumns; } - _transform(chunk, encoding, callback){ - if(this.state.stop === true){ - return; - } - const err = this.__transform(chunk); - if(err !== undefined){ - this.state.stop = true; + return [undefined, columns]; +}; + +const underscore = function(str){ + return str.replace(/([A-Z])/g, function(_, match){ + return '_' + match.toLowerCase(); + }); +}; + +const normalize_options = function(opts) { + const options = {}; + // Merge with user options + for(const opt in opts){ + options[underscore(opt)] = opts[opt]; + } + // Normalize option `bom` + if(options.bom === undefined || options.bom === null || options.bom === false){ + options.bom = false; + }else if(options.bom !== true){ + return [new CsvError('CSV_OPTION_BOOLEAN_INVALID_TYPE', [ + 'option `bom` is optional and must be a boolean value,', + `got ${JSON.stringify(options.bom)}` + ])]; + } + // Normalize option `delimiter` + if(options.delimiter === undefined || options.delimiter === null){ + options.delimiter = ','; + }else if(isBuffer$1(options.delimiter)){ + options.delimiter = options.delimiter.toString(); + }else if(typeof options.delimiter !== 'string'){ + return [new CsvError('CSV_OPTION_DELIMITER_INVALID_TYPE', [ + 'option `delimiter` must be a buffer or a string,', + `got ${JSON.stringify(options.delimiter)}` + ])]; + } + // Normalize option `quote` + if(options.quote === undefined || options.quote === null){ + options.quote = '"'; + }else if(options.quote === true){ + options.quote = '"'; + }else if(options.quote === false){ + options.quote = ''; + }else if (isBuffer$1(options.quote)){ + options.quote = options.quote.toString(); + }else if(typeof options.quote !== 'string'){ + return [new CsvError('CSV_OPTION_QUOTE_INVALID_TYPE', [ + 'option `quote` must be a boolean, a buffer or a string,', + `got ${JSON.stringify(options.quote)}` + ])]; + } + // Normalize option `quoted` + if(options.quoted === undefined || options.quoted === null){ + options.quoted = false; + } + // Normalize option `quoted_empty` + if(options.quoted_empty === undefined || options.quoted_empty === null){ + options.quoted_empty = undefined; + } + // Normalize option `quoted_match` + if(options.quoted_match === undefined || options.quoted_match === null || options.quoted_match === false){ + options.quoted_match = null; + }else if(!Array.isArray(options.quoted_match)){ + options.quoted_match = [options.quoted_match]; + } + if(options.quoted_match){ + for(const quoted_match of options.quoted_match){ + const isString = typeof quoted_match === 'string'; + const isRegExp = quoted_match instanceof RegExp; + if(!isString && !isRegExp){ + return [Error(`Invalid Option: quoted_match must be a string or a regex, got ${JSON.stringify(quoted_match)}`)]; + } } - callback(err); } - _flush(callback){ - if(this.state.stop === true){ - // Note, Node.js 12 call flush even after an error, we must prevent - // `callback` from being called in flush without any error. - return; - } - if(this.info.records === 0){ - this.bom(); - const err = this.headers(); - if(err) callback(err); - } - callback(); + // Normalize option `quoted_string` + if(options.quoted_string === undefined || options.quoted_string === null){ + options.quoted_string = false; + } + // Normalize option `eof` + if(options.eof === undefined || options.eof === null){ + options.eof = true; + } + // Normalize option `escape` + if(options.escape === undefined || options.escape === null){ + options.escape = '"'; + }else if(isBuffer$1(options.escape)){ + options.escape = options.escape.toString(); + }else if(typeof options.escape !== 'string'){ + return [Error(`Invalid Option: escape must be a buffer or a string, got ${JSON.stringify(options.escape)}`)]; + } + if (options.escape.length > 1){ + return [Error(`Invalid Option: escape must be one character, got ${options.escape.length} characters`)]; + } + // Normalize option `header` + if(options.header === undefined || options.header === null){ + options.header = false; + } + // Normalize option `columns` + const [errColumns, columns] = normalize_columns(options.columns); + if(errColumns !== undefined) return [errColumns]; + options.columns = columns; + // Normalize option `quoted` + if(options.quoted === undefined || options.quoted === null){ + options.quoted = false; + } + // Normalize option `cast` + if(options.cast === undefined || options.cast === null){ + options.cast = {}; + } + // Normalize option cast.bigint + if(options.cast.bigint === undefined || options.cast.bigint === null){ + // Cast boolean to string by default + options.cast.bigint = value => '' + value; + } + // Normalize option cast.boolean + if(options.cast.boolean === undefined || options.cast.boolean === null){ + // Cast boolean to string by default + options.cast.boolean = value => value ? '1' : ''; + } + // Normalize option cast.date + if(options.cast.date === undefined || options.cast.date === null){ + // Cast date to timestamp string by default + options.cast.date = value => '' + value.getTime(); + } + // Normalize option cast.number + if(options.cast.number === undefined || options.cast.number === null){ + // Cast number to string using native casting by default + options.cast.number = value => '' + value; + } + // Normalize option cast.object + if(options.cast.object === undefined || options.cast.object === null){ + // Stringify object as JSON by default + options.cast.object = value => JSON.stringify(value); + } + // Normalize option cast.string + if(options.cast.string === undefined || options.cast.string === null){ + // Leave string untouched + options.cast.string = function(value){return value;}; + } + // Normalize option `on_record` + if(options.on_record !== undefined && typeof options.on_record !== 'function'){ + return [Error(`Invalid Option: "on_record" must be a function.`)]; + } + // Normalize option `record_delimiter` + if(options.record_delimiter === undefined || options.record_delimiter === null){ + options.record_delimiter = '\n'; + }else if(isBuffer$1(options.record_delimiter)){ + options.record_delimiter = options.record_delimiter.toString(); + }else if(typeof options.record_delimiter !== 'string'){ + return [Error(`Invalid Option: record_delimiter must be a buffer or a string, got ${JSON.stringify(options.record_delimiter)}`)]; + } + switch(options.record_delimiter){ + case 'auto': + options.record_delimiter = null; + break; + case 'unix': + options.record_delimiter = "\n"; + break; + case 'mac': + options.record_delimiter = "\r"; + break; + case 'windows': + options.record_delimiter = "\r\n"; + break; + case 'ascii': + options.record_delimiter = "\u001e"; + break; + case 'unicode': + options.record_delimiter = "\u2028"; + break; } - __transform(chunk){ - // Chunk validation - if(!Array.isArray(chunk) && typeof chunk !== 'object'){ - return Error(`Invalid Record: expect an array or an object, got ${JSON.stringify(chunk)}`); - } - // Detect columns from the first record - if(this.info.records === 0){ - if(Array.isArray(chunk)){ - if(this.options.header === true && this.options.columns === undefined){ - return Error('Undiscoverable Columns: header option requires column option or object records'); + return [undefined, options]; +}; + +const bom_utf8 = Buffer.from([239, 187, 191]); + +const stringifier = function(options, state, info){ + return { + options: options, + state: state, + info: info, + __transform: function(chunk, push){ + // Chunk validation + if(!Array.isArray(chunk) && typeof chunk !== 'object'){ + return Error(`Invalid Record: expect an array or an object, got ${JSON.stringify(chunk)}`); + } + // Detect columns from the first record + if(this.info.records === 0){ + if(Array.isArray(chunk)){ + if(this.options.header === true && this.options.columns === undefined){ + return Error('Undiscoverable Columns: header option requires column option or object records'); + } + }else if(this.options.columns === undefined){ + const [err, columns] = normalize_columns(Object.keys(chunk)); + if(err) return; + this.options.columns = columns; } - }else if(this.options.columns === undefined){ - const [err, columns] = this.normalize_columns(Object.keys(chunk)); - if(err) return; - this.options.columns = columns; } - } - // Emit the header - if(this.info.records === 0){ - this.bom(); - const err = this.headers(); - if(err) return err; - } - // Emit and stringify the record if an object or an array - try{ - this.emit('record', chunk, this.info.records); - }catch(err){ - return err; - } - // Convert the record into a string - let err, chunk_string; - if(this.options.eof){ - [err, chunk_string] = this.stringify(chunk); - if(err) return err; - if(chunk_string === undefined){ - return; - }else { - chunk_string = chunk_string + this.options.record_delimiter; + // Emit the header + if(this.info.records === 0){ + this.bom(push); + const err = this.headers(push); + if(err) return err; } - }else { - [err, chunk_string] = this.stringify(chunk); - if(err) return err; - if(chunk_string === undefined){ - return; - }else { - if(this.options.header || this.info.records){ - chunk_string = this.options.record_delimiter + chunk_string; + // Emit and stringify the record if an object or an array + try{ + // this.emit('record', chunk, this.info.records); + if(this.options.on_record){ + this.options.on_record(chunk, this.info.records); } + }catch(err){ + return err; } - } - // Emit the csv - this.info.records++; - this.push(chunk_string); - } - stringify(chunk, chunkIsHeader=false){ - if(typeof chunk !== 'object'){ - return [undefined, chunk]; - } - const {columns} = this.options; - const record = []; - // Record is an array - if(Array.isArray(chunk)){ - // We are getting an array but the user has specified output columns. In - // this case, we respect the columns indexes - if(columns){ - chunk.splice(columns.length); - } - // Cast record elements - for(let i=0; i= 0; - const containsQuote = (quote !== '') && value.indexOf(quote) >= 0; - const containsEscape = value.indexOf(escape) >= 0 && (escape !== quote); - const containsRecordDelimiter = value.indexOf(record_delimiter) >= 0; - const quotedString = quoted_string && typeof field === 'string'; - let quotedMatch = quoted_match && quoted_match.filter(quoted_match => { - if(typeof quoted_match === 'string'){ - return value.indexOf(quoted_match) !== -1; - }else { - return quoted_match.test(value); + } + let csvrecord = ''; + for(let i=0; i 0; - const shouldQuote = containsQuote === true || containsdelimiter || containsRecordDelimiter || quoted || quotedString || quotedMatch; - if(shouldQuote === true && containsEscape === true){ - const regexp = escape === '\\' - ? new RegExp(escape + escape, 'g') - : new RegExp(escape, 'g'); - value = value.replace(regexp, escape + escape); + options = {...this.options, ...options}; + [err, options] = normalize_options(options); + if(err !== undefined){ + return [err]; + } + }else if(value === undefined || value === null){ + options = this.options; + }else { + return [Error(`Invalid Casting Value: returned value must return a string, an object, null or undefined, got ${JSON.stringify(value)}`)]; } - if(containsQuote === true){ - const regexp = new RegExp(quote,'g'); - value = value.replace(regexp, escape + quote); + const {delimiter, escape, quote, quoted, quoted_empty, quoted_string, quoted_match, record_delimiter} = options; + if(value){ + if(typeof value !== 'string'){ + return [Error(`Formatter must return a string, null or undefined, got ${JSON.stringify(value)}`)]; + } + const containsdelimiter = delimiter.length && value.indexOf(delimiter) >= 0; + const containsQuote = (quote !== '') && value.indexOf(quote) >= 0; + const containsEscape = value.indexOf(escape) >= 0 && (escape !== quote); + const containsRecordDelimiter = value.indexOf(record_delimiter) >= 0; + const quotedString = quoted_string && typeof field === 'string'; + let quotedMatch = quoted_match && quoted_match.filter(quoted_match => { + if(typeof quoted_match === 'string'){ + return value.indexOf(quoted_match) !== -1; + }else { + return quoted_match.test(value); + } + }); + quotedMatch = quotedMatch && quotedMatch.length > 0; + const shouldQuote = containsQuote === true || containsdelimiter || containsRecordDelimiter || quoted || quotedString || quotedMatch; + if(shouldQuote === true && containsEscape === true){ + const regexp = escape === '\\' + ? new RegExp(escape + escape, 'g') + : new RegExp(escape, 'g'); + value = value.replace(regexp, escape + escape); + } + if(containsQuote === true){ + const regexp = new RegExp(quote,'g'); + value = value.replace(regexp, escape + quote); + } + if(shouldQuote === true){ + value = quote + value + quote; + } + csvrecord += value; + }else if(quoted_empty === true || (field === '' && quoted_string === true && quoted_empty !== false)){ + csvrecord += quote + quote; } - if(shouldQuote === true){ - value = quote + value + quote; + if(i !== record.length - 1){ + csvrecord += delimiter; } - csvrecord += value; - }else if(quoted_empty === true || (field === '' && quoted_string === true && quoted_empty !== false)){ - csvrecord += quote + quote; } - if(i !== record.length - 1){ - csvrecord += delimiter; + return [undefined, csvrecord]; + }, + bom: function(push){ + if(this.options.bom !== true){ + return; + } + push(bom_utf8); + }, + headers: function(push){ + if(this.options.header === false){ + return; + } + if(this.options.columns === undefined){ + return; + } + let err; + let headers = this.options.columns.map(column => column.header); + if(this.options.eof){ + [err, headers] = this.stringify(headers, true); + headers += this.options.record_delimiter; + }else { + [err, headers] = this.stringify(headers); + } + if(err) return err; + push(headers); + }, + __cast: function(value, context){ + const type = typeof value; + try{ + if(type === 'string'){ // Fine for 99% of the cases + return [undefined, this.options.cast.string(value, context)]; + }else if(type === 'bigint'){ + return [undefined, this.options.cast.bigint(value, context)]; + }else if(type === 'number'){ + return [undefined, this.options.cast.number(value, context)]; + }else if(type === 'boolean'){ + return [undefined, this.options.cast.boolean(value, context)]; + }else if(value instanceof Date){ + return [undefined, this.options.cast.date(value, context)]; + }else if(type === 'object' && value !== null){ + return [undefined, this.options.cast.object(value, context)]; + }else { + return [undefined, value, value]; + } + }catch(err){ + return [err]; } } - return [undefined, csvrecord]; - } - bom(){ - if(this.options.bom !== true){ - return; - } - this.push(bom_utf8); + }; +}; + +class Stringifier extends Transform { + constructor(opts = {}){ + super({...{writableObjectMode: true}, ...opts}); + const [err, options] = normalize_options(opts); + if(err !== undefined) throw err; + // Expose options + this.options = options; + // Internal state + this.state = { + stop: false + }; + // Information + this.info = { + records: 0 + }; + this.api = stringifier(this.options, this.state, this.info); + this.api.options.on_record = (...args) => { + this.emit('record', ...args); + }; } - headers(){ - if(this.options.header === false){ - return; - } - if(this.options.columns === undefined){ + _transform(chunk, encoding, callback){ + if(this.state.stop === true){ return; } - let err; - let headers = this.options.columns.map(column => column.header); - if(this.options.eof){ - [err, headers] = this.stringify(headers, true); - headers += this.options.record_delimiter; - }else { - [err, headers] = this.stringify(headers); - } - if(err) return err; - this.push(headers); - } - __cast(value, context){ - const type = typeof value; - try{ - if(type === 'string'){ // Fine for 99% of the cases - return [undefined, this.options.cast.string(value, context)]; - }else if(type === 'bigint'){ - return [undefined, this.options.cast.bigint(value, context)]; - }else if(type === 'number'){ - return [undefined, this.options.cast.number(value, context)]; - }else if(type === 'boolean'){ - return [undefined, this.options.cast.boolean(value, context)]; - }else if(value instanceof Date){ - return [undefined, this.options.cast.date(value, context)]; - }else if(type === 'object' && value !== null){ - return [undefined, this.options.cast.object(value, context)]; - }else { - return [undefined, value, value]; - } - }catch(err){ - return [err]; + const err = this.api.__transform(chunk, this.push.bind(this)); + if(err !== undefined){ + this.state.stop = true; } + callback(err); } - normalize_columns(columns){ - if(columns === undefined || columns === null){ - return []; - } - if(typeof columns !== 'object'){ - return [Error('Invalid option "columns": expect an array or an object')]; + _flush(callback){ + if(this.state.stop === true){ + // Note, Node.js 12 call flush even after an error, we must prevent + // `callback` from being called in flush without any error. + return; } - if(!Array.isArray(columns)){ - const newcolumns = []; - for(const k in columns){ - newcolumns.push({ - key: k, - header: columns[k] - }); - } - columns = newcolumns; - }else { - const newcolumns = []; - for(const column of columns){ - if(typeof column === 'string'){ - newcolumns.push({ - key: column, - header: column - }); - }else if(typeof column === 'object' && column !== undefined && !Array.isArray(column)){ - if(!column.key){ - return [Error('Invalid column definition: property "key" is required')]; - } - if(column.header === undefined){ - column.header = column.key; - } - newcolumns.push(column); - }else { - return [Error('Invalid column definition: expect a string or an object')]; - } - } - columns = newcolumns; + if(this.info.records === 0){ + this.api.bom(this.push.bind(this)); + const err = this.api.headers(this.push.bind(this)); + if(err) callback(err); } - return [undefined, columns]; + callback(); } } @@ -7273,7 +7360,7 @@ const stringify = function(){ const type = typeof argument; if(data === undefined && (Array.isArray(argument))){ data = argument; - }else if(options === undefined && isObject(argument)){ + }else if(options === undefined && is_object(argument)){ options = argument; }else if(callback === undefined && type === 'function'){ callback = argument; diff --git a/packages/csv/dist/esm/sync.js b/packages/csv/dist/esm/sync.js index 5dc6b198f..c438b78e7 100644 --- a/packages/csv/dist/esm/sync.js +++ b/packages/csv/dist/esm/sync.js @@ -112,7 +112,7 @@ function fromByteArray (uint8) { return parts.join('') } -function read (buffer, offset, isLE, mLen, nBytes) { +function read$1 (buffer, offset, isLE, mLen, nBytes) { var e, m; var eLen = nBytes * 8 - mLen - 1; var eMax = (1 << eLen) - 1; @@ -1393,22 +1393,22 @@ Buffer.prototype.readInt32BE = function readInt32BE (offset, noAssert) { Buffer.prototype.readFloatLE = function readFloatLE (offset, noAssert) { if (!noAssert) checkOffset(offset, 4, this.length); - return read(this, offset, true, 23, 4) + return read$1(this, offset, true, 23, 4) }; Buffer.prototype.readFloatBE = function readFloatBE (offset, noAssert) { if (!noAssert) checkOffset(offset, 4, this.length); - return read(this, offset, false, 23, 4) + return read$1(this, offset, false, 23, 4) }; Buffer.prototype.readDoubleLE = function readDoubleLE (offset, noAssert) { if (!noAssert) checkOffset(offset, 8, this.length); - return read(this, offset, true, 52, 8) + return read$1(this, offset, true, 52, 8) }; Buffer.prototype.readDoubleBE = function readDoubleBE (offset, noAssert) { if (!noAssert) checkOffset(offset, 8, this.length); - return read(this, offset, false, 52, 8) + return read$1(this, offset, false, 52, 8) }; function checkInt (buf, value, offset, ext, max, min) { @@ -2627,7 +2627,7 @@ function format(f) { } }); for (var x = args[i]; i < len; x = args[++i]) { - if (isNull(x) || !isObject$2(x)) { + if (isNull(x) || !isObject(x)) { str += ' ' + x; } else { str += ' ' + inspect(x); @@ -3043,19 +3043,19 @@ function isUndefined(arg) { } function isRegExp(re) { - return isObject$2(re) && objectToString(re) === '[object RegExp]'; + return isObject(re) && objectToString(re) === '[object RegExp]'; } -function isObject$2(arg) { +function isObject(arg) { return typeof arg === 'object' && arg !== null; } function isDate(d) { - return isObject$2(d) && objectToString(d) === '[object Date]'; + return isObject(d) && objectToString(d) === '[object Date]'; } function isError(e) { - return isObject$2(e) && + return isObject(e) && (objectToString(e) === '[object Error]' || e instanceof Error); } @@ -3106,7 +3106,7 @@ function log() { function _extend(origin, add) { // Don't do anything if add isn't an object - if (!add || !isObject$2(add)) return origin; + if (!add || !isObject(add)) return origin; var keys = Object.keys(add); var i = keys.length; @@ -3128,7 +3128,7 @@ var util = { isFunction: isFunction, isError: isError, isDate: isDate, - isObject: isObject$2, + isObject: isObject, isRegExp: isRegExp, isUndefined: isUndefined, isSymbol: isSymbol$1, @@ -5036,20 +5036,64 @@ Stream.prototype.pipe = function(dest, options) { return dest; }; -const Generator = function(options = {}){ +const init_state$1 = (options) => { + // State + return { + start_time: options.duration ? Date.now() : null, + fixed_size_buffer: '', + count_written: 0, + count_created: 0, + }; +}; + +// Generate a random number between 0 and 1 with 2 decimals. The function is idempotent if it detect the "seed" option. +const random = function(options={}){ + if(options.seed){ + return options.seed = options.seed * Math.PI * 100 % 100 / 100; + }else { + return Math.random(); + } +}; + +const types = { + // Generate an ASCII value. + ascii: function({options}){ + const column = []; + const nb_chars = Math.ceil(random(options) * options.maxWordLength); + for(let i=0; i { // Convert Stream Readable options if underscored - if(options.high_water_mark){ - options.highWaterMark = options.high_water_mark; + if(opts.high_water_mark){ + opts.highWaterMark = opts.high_water_mark; } - if(options.object_mode){ - options.objectMode = options.object_mode; + if(opts.object_mode){ + opts.objectMode = opts.object_mode; } - // Call parent constructor - Stream.Readable.call(this, options); // Clone and camelize options - this.options = {}; - for(const k in options){ - this.options[Generator.camelize(k)] = options[k]; + const options = {}; + for(const k in opts){ + options[camelize(k)] = opts[k]; } // Normalize options const dft = { @@ -5067,110 +5111,100 @@ const Generator = function(options = {}){ sleep: 0, }; for(const k in dft){ - if(this.options[k] === undefined){ - this.options[k] = dft[k]; + if(options[k] === undefined){ + options[k] = dft[k]; } } // Default values - if(this.options.eof === true){ - this.options.eof = this.options.rowDelimiter; + if(options.eof === true){ + options.eof = options.rowDelimiter; } - // State - this._ = { - start_time: this.options.duration ? Date.now() : null, - fixed_size_buffer: '', - count_written: 0, - count_created: 0, - }; - if(typeof this.options.columns === 'number'){ - this.options.columns = new Array(this.options.columns); + if(typeof options.columns === 'number'){ + options.columns = new Array(options.columns); } - const accepted_header_types = Object.keys(Generator).filter((t) => (!['super_', 'camelize'].includes(t))); - for(let i = 0; i < this.options.columns.length; i++){ - const v = this.options.columns[i] || 'ascii'; + const accepted_header_types = Object.keys(types).filter((t) => (!['super_', 'camelize'].includes(t))); + for(let i = 0; i < options.columns.length; i++){ + const v = options.columns[i] || 'ascii'; if(typeof v === 'string'){ if(!accepted_header_types.includes(v)){ throw Error(`Invalid column type: got "${v}", default values are ${JSON.stringify(accepted_header_types)}`); } - this.options.columns[i] = Generator[v]; + options.columns[i] = types[v]; } } - return this; + return options; }; -util.inherits(Generator, Stream.Readable); -// Generate a random number between 0 and 1 with 2 decimals. The function is idempotent if it detect the "seed" option. -Generator.prototype.random = function(){ - if(this.options.seed){ - return this.options.seed = this.options.seed * Math.PI * 100 % 100 / 100; - }else { - return Math.random(); - } -}; -// Stop the generation. -Generator.prototype.end = function(){ - this.push(null); -}; -// Put new data into the read queue. -Generator.prototype._read = function(size){ +const read = (options, state, size, push, close) => { // Already started const data = []; - let length = this._.fixed_size_buffer.length; + let length = state.fixed_size_buffer.length; if(length !== 0){ - data.push(this._.fixed_size_buffer); + data.push(state.fixed_size_buffer); } // eslint-disable-next-line while(true){ // Time for some rest: flush first and stop later - if((this._.count_created === this.options.length) || (this.options.end && Date.now() > this.options.end) || (this.options.duration && Date.now() > this._.start_time + this.options.duration)){ + if((state.count_created === options.length) || (options.end && Date.now() > options.end) || (options.duration && Date.now() > state.start_time + options.duration)){ // Flush if(data.length){ - if(this.options.objectMode){ + if(options.objectMode){ for(const record of data){ - this.__push(record); + push(record); } }else { - this.__push(data.join('') + (this.options.eof ? this.options.eof : '')); + push(data.join('') + (options.eof ? options.eof : '')); } - this._.end = true; + state.end = true; }else { - this.push(null); + close(); } return; } // Create the record let record = []; let recordLength; - this.options.columns.forEach((fn) => { - record.push(fn(this)); + options.columns.forEach((fn) => { + const result = fn({options: options, state: state}); + const type = typeof result; + if(result !== null && type !== 'string' && type !== 'number'){ + throw Error([ + 'INVALID_VALUE:', + 'values returned by column function must be', + 'a string, a number or null,', + `got ${JSON.stringify(result)}` + ].join(' ')); + } + record.push(result); }); // Obtain record length - if(this.options.objectMode){ + if(options.objectMode){ recordLength = 0; // recordLength is currently equal to the number of columns // This is wrong and shall equal to 1 record only - for(const column of record) + for(const column of record){ recordLength += column.length; + } }else { // Stringify the record - record = (this._.count_created === 0 ? '' : this.options.rowDelimiter)+record.join(this.options.delimiter); + record = (state.count_created === 0 ? '' : options.rowDelimiter)+record.join(options.delimiter); recordLength = record.length; } - this._.count_created++; + state.count_created++; if(length + recordLength > size){ - if(this.options.objectMode){ + if(options.objectMode){ data.push(record); for(const record of data){ - this.__push(record); + push(record); } }else { - if(this.options.fixedSize){ - this._.fixed_size_buffer = record.substr(size - length); + if(options.fixedSize){ + state.fixed_size_buffer = record.substr(size - length); data.push(record.substr(0, size - length)); }else { data.push(record); } - this.__push(data.join('')); + push(data.join('')); } return; } @@ -5178,42 +5212,41 @@ Generator.prototype._read = function(size){ data.push(record); } }; + +const Generator = function(options = {}){ + this.options = normalize_options$2(options); + // Call parent constructor + Stream.Readable.call(this, this.options); + this.state = init_state$1(this.options); + return this; +}; +util.inherits(Generator, Stream.Readable); + +// Stop the generation. +Generator.prototype.end = function(){ + this.push(null); +}; +// Put new data into the read queue. +Generator.prototype._read = function(size){ + const self = this; + read(this.options, this.state, size, function(chunk) { + self.__push(chunk); + }, function(){ + self.push(null); + }); +}; // Put new data into the read queue. Generator.prototype.__push = function(record){ + // console.log('push', record) const push = () => { - this._.count_written++; + this.state.count_written++; this.push(record); - if(this._.end === true){ + if(this.state.end === true){ return this.push(null); } }; this.options.sleep > 0 ? setTimeout(push, this.options.sleep) : push(); }; -// Generate an ASCII value. -Generator.ascii = function(gen){ - // Column - const column = []; - const nb_chars = Math.ceil(gen.random() * gen.options.maxWordLength); - for(let i=0; i delimiter.length), + // Skip if the remaining buffer can be escape sequence + options.quote !== null ? options.quote.length : 0, + ), + previousBuf: undefined, + quoting: false, + stop: false, + rawBuffer: new ResizeableBuffer(100), + record: [], + recordHasError: false, + record_length: 0, + recordDelimiterMaxLength: options.record_delimiter.length === 0 ? 2 : Math.max(...options.record_delimiter.map((v) => v.length)), + trimChars: [Buffer.from(' ', options.encoding)[0], Buffer.from('\t', options.encoding)[0]], + wasQuoting: false, + wasRowDelimiter: false + }; }; -class CsvError$1 extends Error { - constructor(code, message, options, ...contexts) { - if(Array.isArray(message)) message = message.join(' '); - super(message); - if(Error.captureStackTrace !== undefined){ - Error.captureStackTrace(this, CsvError$1); - } - this.code = code; - for(const context of contexts){ - for(const key in context){ - const value = context[key]; - this[key] = isBuffer$1(value) ? value.toString(options.encoding) : value == null ? value : JSON.parse(JSON.stringify(value)); - } - } - } -} - const underscore$1 = function(str){ return str.replace(/([A-Z])/g, function(_, match){ return '_' + match.toLowerCase(); }); }; -const isObject$1 = function(obj){ - return (typeof obj === 'object' && obj !== null && !Array.isArray(obj)); -}; - -const isRecordEmpty = function(record){ - return record.every((field) => field == null || field.toString && field.toString().trim() === ''); -}; - -const normalizeColumnsArray = function(columns){ - const normalizedColumns = []; - for(let i = 0, l = columns.length; i < l; i++){ - const column = columns[i]; - if(column === undefined || column === null || column === false){ - normalizedColumns[i] = { disabled: true }; - }else if(typeof column === 'string'){ - normalizedColumns[i] = { name: column }; - }else if(isObject$1(column)){ - if(typeof column.name !== 'string'){ - throw new CsvError$1('CSV_OPTION_COLUMNS_MISSING_NAME', [ - 'Option columns missing name:', - `property "name" is required at position ${i}`, - 'when column is an object literal' - ]); - } - normalizedColumns[i] = column; - }else { - throw new CsvError$1('CSV_INVALID_COLUMN_DEFINITION', [ - 'Invalid column definition:', - 'expect a string or a literal object,', - `got ${JSON.stringify(column)} at position ${i}` - ]); - } - } - return normalizedColumns; -}; - -class Parser extends Transform { - constructor(opts = {}){ - super({...{readableObjectMode: true}, ...opts, encoding: null}); - this.__originalOptions = opts; - this.__normalizeOptions(opts); - } - __normalizeOptions(opts){ - const options = {}; - // Merge with user options - for(const opt in opts){ - options[underscore$1(opt)] = opts[opt]; - } - // Normalize option `encoding` - // Note: defined first because other options depends on it - // to convert chars/strings into buffers. - if(options.encoding === undefined || options.encoding === true){ - options.encoding = 'utf8'; - }else if(options.encoding === null || options.encoding === false){ - options.encoding = null; - }else if(typeof options.encoding !== 'string' && options.encoding !== null){ - throw new CsvError$1('CSV_INVALID_OPTION_ENCODING', [ - 'Invalid option encoding:', - 'encoding must be a string or null to return a buffer,', - `got ${JSON.stringify(options.encoding)}` - ], options); - } - // Normalize option `bom` - if(options.bom === undefined || options.bom === null || options.bom === false){ - options.bom = false; - }else if(options.bom !== true){ - throw new CsvError$1('CSV_INVALID_OPTION_BOM', [ - 'Invalid option bom:', 'bom must be true,', - `got ${JSON.stringify(options.bom)}` - ], options); - } - // Normalize option `cast` - let fnCastField = null; - if(options.cast === undefined || options.cast === null || options.cast === false || options.cast === ''){ - options.cast = undefined; - }else if(typeof options.cast === 'function'){ - fnCastField = options.cast; - options.cast = true; - }else if(options.cast !== true){ - throw new CsvError$1('CSV_INVALID_OPTION_CAST', [ - 'Invalid option cast:', 'cast must be true or a function,', - `got ${JSON.stringify(options.cast)}` - ], options); - } - // Normalize option `cast_date` - if(options.cast_date === undefined || options.cast_date === null || options.cast_date === false || options.cast_date === ''){ - options.cast_date = false; - }else if(options.cast_date === true){ - options.cast_date = function(value){ - const date = Date.parse(value); - return !isNaN(date) ? new Date(date) : value; - }; - }else { - throw new CsvError$1('CSV_INVALID_OPTION_CAST_DATE', [ - 'Invalid option cast_date:', 'cast_date must be true or a function,', - `got ${JSON.stringify(options.cast_date)}` - ], options); - } - // Normalize option `columns` - let fnFirstLineToHeaders = null; - if(options.columns === true){ - // Fields in the first line are converted as-is to columns - fnFirstLineToHeaders = undefined; - }else if(typeof options.columns === 'function'){ - fnFirstLineToHeaders = options.columns; - options.columns = true; - }else if(Array.isArray(options.columns)){ - options.columns = normalizeColumnsArray(options.columns); - }else if(options.columns === undefined || options.columns === null || options.columns === false){ - options.columns = false; - }else { - throw new CsvError$1('CSV_INVALID_OPTION_COLUMNS', [ - 'Invalid option columns:', - 'expect an array, a function or true,', - `got ${JSON.stringify(options.columns)}` +const normalize_options$1 = function(opts){ + const options = {}; + // Merge with user options + for(const opt in opts){ + options[underscore$1(opt)] = opts[opt]; + } + // Normalize option `encoding` + // Note: defined first because other options depends on it + // to convert chars/strings into buffers. + if(options.encoding === undefined || options.encoding === true){ + options.encoding = 'utf8'; + }else if(options.encoding === null || options.encoding === false){ + options.encoding = null; + }else if(typeof options.encoding !== 'string' && options.encoding !== null){ + throw new CsvError$1('CSV_INVALID_OPTION_ENCODING', [ + 'Invalid option encoding:', + 'encoding must be a string or null to return a buffer,', + `got ${JSON.stringify(options.encoding)}` + ], options); + } + // Normalize option `bom` + if(options.bom === undefined || options.bom === null || options.bom === false){ + options.bom = false; + }else if(options.bom !== true){ + throw new CsvError$1('CSV_INVALID_OPTION_BOM', [ + 'Invalid option bom:', 'bom must be true,', + `got ${JSON.stringify(options.bom)}` + ], options); + } + // Normalize option `cast` + options.cast_function = null; + if(options.cast === undefined || options.cast === null || options.cast === false || options.cast === ''){ + options.cast = undefined; + }else if(typeof options.cast === 'function'){ + options.cast_function = options.cast; + options.cast = true; + }else if(options.cast !== true){ + throw new CsvError$1('CSV_INVALID_OPTION_CAST', [ + 'Invalid option cast:', 'cast must be true or a function,', + `got ${JSON.stringify(options.cast)}` + ], options); + } + // Normalize option `cast_date` + if(options.cast_date === undefined || options.cast_date === null || options.cast_date === false || options.cast_date === ''){ + options.cast_date = false; + }else if(options.cast_date === true){ + options.cast_date = function(value){ + const date = Date.parse(value); + return !isNaN(date) ? new Date(date) : value; + }; + }else { + throw new CsvError$1('CSV_INVALID_OPTION_CAST_DATE', [ + 'Invalid option cast_date:', 'cast_date must be true or a function,', + `got ${JSON.stringify(options.cast_date)}` + ], options); + } + // Normalize option `columns` + options.cast_first_line_to_header = null; + if(options.columns === true){ + // Fields in the first line are converted as-is to columns + options.cast_first_line_to_header = undefined; + }else if(typeof options.columns === 'function'){ + options.cast_first_line_to_header = options.columns; + options.columns = true; + }else if(Array.isArray(options.columns)){ + options.columns = normalize_columns_array(options.columns); + }else if(options.columns === undefined || options.columns === null || options.columns === false){ + options.columns = false; + }else { + throw new CsvError$1('CSV_INVALID_OPTION_COLUMNS', [ + 'Invalid option columns:', + 'expect an array, a function or true,', + `got ${JSON.stringify(options.columns)}` + ], options); + } + // Normalize option `group_columns_by_name` + if(options.group_columns_by_name === undefined || options.group_columns_by_name === null || options.group_columns_by_name === false){ + options.group_columns_by_name = false; + }else if(options.group_columns_by_name !== true){ + throw new CsvError$1('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ + 'Invalid option group_columns_by_name:', + 'expect an boolean,', + `got ${JSON.stringify(options.group_columns_by_name)}` + ], options); + }else if(options.columns === false){ + throw new CsvError$1('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ + 'Invalid option group_columns_by_name:', + 'the `columns` mode must be activated.' + ], options); + } + // Normalize option `comment` + if(options.comment === undefined || options.comment === null || options.comment === false || options.comment === ''){ + options.comment = null; + }else { + if(typeof options.comment === 'string'){ + options.comment = Buffer.from(options.comment, options.encoding); + } + if(!isBuffer$1(options.comment)){ + throw new CsvError$1('CSV_INVALID_OPTION_COMMENT', [ + 'Invalid option comment:', + 'comment must be a buffer or a string,', + `got ${JSON.stringify(options.comment)}` ], options); } - // Normalize option `group_columns_by_name` - if(options.group_columns_by_name === undefined || options.group_columns_by_name === null || options.group_columns_by_name === false){ - options.group_columns_by_name = false; - }else if(options.group_columns_by_name !== true){ - throw new CsvError$1('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ - 'Invalid option group_columns_by_name:', - 'expect an boolean,', - `got ${JSON.stringify(options.group_columns_by_name)}` - ], options); - }else if(options.columns === false){ - throw new CsvError$1('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ - 'Invalid option group_columns_by_name:', - 'the `columns` mode must be activated.' - ], options); + } + // Normalize option `delimiter` + const delimiter_json = JSON.stringify(options.delimiter); + if(!Array.isArray(options.delimiter)) options.delimiter = [options.delimiter]; + if(options.delimiter.length === 0){ + throw new CsvError$1('CSV_INVALID_OPTION_DELIMITER', [ + 'Invalid option delimiter:', + 'delimiter must be a non empty string or buffer or array of string|buffer,', + `got ${delimiter_json}` + ], options); + } + options.delimiter = options.delimiter.map(function(delimiter){ + if(delimiter === undefined || delimiter === null || delimiter === false){ + return Buffer.from(',', options.encoding); } - // Normalize option `comment` - if(options.comment === undefined || options.comment === null || options.comment === false || options.comment === ''){ - options.comment = null; - }else { - if(typeof options.comment === 'string'){ - options.comment = Buffer.from(options.comment, options.encoding); - } - if(!isBuffer$1(options.comment)){ - throw new CsvError$1('CSV_INVALID_OPTION_COMMENT', [ - 'Invalid option comment:', - 'comment must be a buffer or a string,', - `got ${JSON.stringify(options.comment)}` - ], options); - } + if(typeof delimiter === 'string'){ + delimiter = Buffer.from(delimiter, options.encoding); } - // Normalize option `delimiter` - const delimiter_json = JSON.stringify(options.delimiter); - if(!Array.isArray(options.delimiter)) options.delimiter = [options.delimiter]; - if(options.delimiter.length === 0){ + if(!isBuffer$1(delimiter) || delimiter.length === 0){ throw new CsvError$1('CSV_INVALID_OPTION_DELIMITER', [ 'Invalid option delimiter:', 'delimiter must be a non empty string or buffer or array of string|buffer,', `got ${delimiter_json}` ], options); } - options.delimiter = options.delimiter.map(function(delimiter){ - if(delimiter === undefined || delimiter === null || delimiter === false){ - return Buffer.from(',', options.encoding); - } - if(typeof delimiter === 'string'){ - delimiter = Buffer.from(delimiter, options.encoding); - } - if(!isBuffer$1(delimiter) || delimiter.length === 0){ - throw new CsvError$1('CSV_INVALID_OPTION_DELIMITER', [ - 'Invalid option delimiter:', - 'delimiter must be a non empty string or buffer or array of string|buffer,', - `got ${delimiter_json}` - ], options); - } - return delimiter; - }); - // Normalize option `escape` - if(options.escape === undefined || options.escape === true){ - options.escape = Buffer.from('"', options.encoding); - }else if(typeof options.escape === 'string'){ - options.escape = Buffer.from(options.escape, options.encoding); - }else if (options.escape === null || options.escape === false){ - options.escape = null; - } - if(options.escape !== null){ - if(!isBuffer$1(options.escape)){ - throw new Error(`Invalid Option: escape must be a buffer, a string or a boolean, got ${JSON.stringify(options.escape)}`); - } + return delimiter; + }); + // Normalize option `escape` + if(options.escape === undefined || options.escape === true){ + options.escape = Buffer.from('"', options.encoding); + }else if(typeof options.escape === 'string'){ + options.escape = Buffer.from(options.escape, options.encoding); + }else if (options.escape === null || options.escape === false){ + options.escape = null; + } + if(options.escape !== null){ + if(!isBuffer$1(options.escape)){ + throw new Error(`Invalid Option: escape must be a buffer, a string or a boolean, got ${JSON.stringify(options.escape)}`); + } + } + // Normalize option `from` + if(options.from === undefined || options.from === null){ + options.from = 1; + }else { + if(typeof options.from === 'string' && /\d+/.test(options.from)){ + options.from = parseInt(options.from); } - // Normalize option `from` - if(options.from === undefined || options.from === null){ - options.from = 1; - }else { - if(typeof options.from === 'string' && /\d+/.test(options.from)){ - options.from = parseInt(options.from); - } - if(Number.isInteger(options.from)){ - if(options.from < 0){ - throw new Error(`Invalid Option: from must be a positive integer, got ${JSON.stringify(opts.from)}`); - } - }else { - throw new Error(`Invalid Option: from must be an integer, got ${JSON.stringify(options.from)}`); + if(Number.isInteger(options.from)){ + if(options.from < 0){ + throw new Error(`Invalid Option: from must be a positive integer, got ${JSON.stringify(opts.from)}`); } - } - // Normalize option `from_line` - if(options.from_line === undefined || options.from_line === null){ - options.from_line = 1; }else { - if(typeof options.from_line === 'string' && /\d+/.test(options.from_line)){ - options.from_line = parseInt(options.from_line); - } - if(Number.isInteger(options.from_line)){ - if(options.from_line <= 0){ - throw new Error(`Invalid Option: from_line must be a positive integer greater than 0, got ${JSON.stringify(opts.from_line)}`); - } - }else { - throw new Error(`Invalid Option: from_line must be an integer, got ${JSON.stringify(opts.from_line)}`); - } - } - // Normalize options `ignore_last_delimiters` - if(options.ignore_last_delimiters === undefined || options.ignore_last_delimiters === null){ - options.ignore_last_delimiters = false; - }else if(typeof options.ignore_last_delimiters === 'number'){ - options.ignore_last_delimiters = Math.floor(options.ignore_last_delimiters); - if(options.ignore_last_delimiters === 0){ - options.ignore_last_delimiters = false; - } - }else if(typeof options.ignore_last_delimiters !== 'boolean'){ - throw new CsvError$1('CSV_INVALID_OPTION_IGNORE_LAST_DELIMITERS', [ - 'Invalid option `ignore_last_delimiters`:', - 'the value must be a boolean value or an integer,', - `got ${JSON.stringify(options.ignore_last_delimiters)}` - ], options); + throw new Error(`Invalid Option: from must be an integer, got ${JSON.stringify(options.from)}`); } - if(options.ignore_last_delimiters === true && options.columns === false){ - throw new CsvError$1('CSV_IGNORE_LAST_DELIMITERS_REQUIRES_COLUMNS', [ - 'The option `ignore_last_delimiters`', - 'requires the activation of the `columns` option' - ], options); + } + // Normalize option `from_line` + if(options.from_line === undefined || options.from_line === null){ + options.from_line = 1; + }else { + if(typeof options.from_line === 'string' && /\d+/.test(options.from_line)){ + options.from_line = parseInt(options.from_line); } - // Normalize option `info` - if(options.info === undefined || options.info === null || options.info === false){ - options.info = false; - }else if(options.info !== true){ - throw new Error(`Invalid Option: info must be true, got ${JSON.stringify(options.info)}`); - } - // Normalize option `max_record_size` - if(options.max_record_size === undefined || options.max_record_size === null || options.max_record_size === false){ - options.max_record_size = 0; - }else if(Number.isInteger(options.max_record_size) && options.max_record_size >= 0);else if(typeof options.max_record_size === 'string' && /\d+/.test(options.max_record_size)){ - options.max_record_size = parseInt(options.max_record_size); - }else { - throw new Error(`Invalid Option: max_record_size must be a positive integer, got ${JSON.stringify(options.max_record_size)}`); - } - // Normalize option `objname` - if(options.objname === undefined || options.objname === null || options.objname === false){ - options.objname = undefined; - }else if(isBuffer$1(options.objname)){ - if(options.objname.length === 0){ - throw new Error(`Invalid Option: objname must be a non empty buffer`); - } - if(options.encoding === null);else { - options.objname = options.objname.toString(options.encoding); - } - }else if(typeof options.objname === 'string'){ - if(options.objname.length === 0){ - throw new Error(`Invalid Option: objname must be a non empty string`); - } - // Great, nothing to do - }else if(typeof options.objname === 'number');else { - throw new Error(`Invalid Option: objname must be a string or a buffer, got ${options.objname}`); - } - if(options.objname !== undefined){ - if(typeof options.objname === 'number'){ - if(options.columns !== false){ - throw Error('Invalid Option: objname index cannot be combined with columns or be defined as a field'); - } - }else { // A string or a buffer - if(options.columns === false){ - throw Error('Invalid Option: objname field must be combined with columns or be defined as an index'); - } + if(Number.isInteger(options.from_line)){ + if(options.from_line <= 0){ + throw new Error(`Invalid Option: from_line must be a positive integer greater than 0, got ${JSON.stringify(opts.from_line)}`); } + }else { + throw new Error(`Invalid Option: from_line must be an integer, got ${JSON.stringify(opts.from_line)}`); } - // Normalize option `on_record` - if(options.on_record === undefined || options.on_record === null){ - options.on_record = undefined; - }else if(typeof options.on_record !== 'function'){ - throw new CsvError$1('CSV_INVALID_OPTION_ON_RECORD', [ - 'Invalid option `on_record`:', - 'expect a function,', - `got ${JSON.stringify(options.on_record)}` - ], options); + } + // Normalize options `ignore_last_delimiters` + if(options.ignore_last_delimiters === undefined || options.ignore_last_delimiters === null){ + options.ignore_last_delimiters = false; + }else if(typeof options.ignore_last_delimiters === 'number'){ + options.ignore_last_delimiters = Math.floor(options.ignore_last_delimiters); + if(options.ignore_last_delimiters === 0){ + options.ignore_last_delimiters = false; } - // Normalize option `quote` - if(options.quote === null || options.quote === false || options.quote === ''){ - options.quote = null; - }else { - if(options.quote === undefined || options.quote === true){ - options.quote = Buffer.from('"', options.encoding); - }else if(typeof options.quote === 'string'){ - options.quote = Buffer.from(options.quote, options.encoding); + }else if(typeof options.ignore_last_delimiters !== 'boolean'){ + throw new CsvError$1('CSV_INVALID_OPTION_IGNORE_LAST_DELIMITERS', [ + 'Invalid option `ignore_last_delimiters`:', + 'the value must be a boolean value or an integer,', + `got ${JSON.stringify(options.ignore_last_delimiters)}` + ], options); + } + if(options.ignore_last_delimiters === true && options.columns === false){ + throw new CsvError$1('CSV_IGNORE_LAST_DELIMITERS_REQUIRES_COLUMNS', [ + 'The option `ignore_last_delimiters`', + 'requires the activation of the `columns` option' + ], options); + } + // Normalize option `info` + if(options.info === undefined || options.info === null || options.info === false){ + options.info = false; + }else if(options.info !== true){ + throw new Error(`Invalid Option: info must be true, got ${JSON.stringify(options.info)}`); + } + // Normalize option `max_record_size` + if(options.max_record_size === undefined || options.max_record_size === null || options.max_record_size === false){ + options.max_record_size = 0; + }else if(Number.isInteger(options.max_record_size) && options.max_record_size >= 0);else if(typeof options.max_record_size === 'string' && /\d+/.test(options.max_record_size)){ + options.max_record_size = parseInt(options.max_record_size); + }else { + throw new Error(`Invalid Option: max_record_size must be a positive integer, got ${JSON.stringify(options.max_record_size)}`); + } + // Normalize option `objname` + if(options.objname === undefined || options.objname === null || options.objname === false){ + options.objname = undefined; + }else if(isBuffer$1(options.objname)){ + if(options.objname.length === 0){ + throw new Error(`Invalid Option: objname must be a non empty buffer`); + } + if(options.encoding === null);else { + options.objname = options.objname.toString(options.encoding); + } + }else if(typeof options.objname === 'string'){ + if(options.objname.length === 0){ + throw new Error(`Invalid Option: objname must be a non empty string`); + } + // Great, nothing to do + }else if(typeof options.objname === 'number');else { + throw new Error(`Invalid Option: objname must be a string or a buffer, got ${options.objname}`); + } + if(options.objname !== undefined){ + if(typeof options.objname === 'number'){ + if(options.columns !== false){ + throw Error('Invalid Option: objname index cannot be combined with columns or be defined as a field'); } - if(!isBuffer$1(options.quote)){ - throw new Error(`Invalid Option: quote must be a buffer or a string, got ${JSON.stringify(options.quote)}`); + }else { // A string or a buffer + if(options.columns === false){ + throw Error('Invalid Option: objname field must be combined with columns or be defined as an index'); } } - // Normalize option `raw` - if(options.raw === undefined || options.raw === null || options.raw === false){ - options.raw = false; - }else if(options.raw !== true){ - throw new Error(`Invalid Option: raw must be true, got ${JSON.stringify(options.raw)}`); - } - // Normalize option `record_delimiter` - if(options.record_delimiter === undefined){ - options.record_delimiter = []; - }else if(typeof options.record_delimiter === 'string' || isBuffer$1(options.record_delimiter)){ - if(options.record_delimiter.length === 0){ - throw new CsvError$1('CSV_INVALID_OPTION_RECORD_DELIMITER', [ - 'Invalid option `record_delimiter`:', - 'value must be a non empty string or buffer,', - `got ${JSON.stringify(options.record_delimiter)}` - ], options); - } - options.record_delimiter = [options.record_delimiter]; - }else if(!Array.isArray(options.record_delimiter)){ + } + // Normalize option `on_record` + if(options.on_record === undefined || options.on_record === null){ + options.on_record = undefined; + }else if(typeof options.on_record !== 'function'){ + throw new CsvError$1('CSV_INVALID_OPTION_ON_RECORD', [ + 'Invalid option `on_record`:', + 'expect a function,', + `got ${JSON.stringify(options.on_record)}` + ], options); + } + // Normalize option `quote` + if(options.quote === null || options.quote === false || options.quote === ''){ + options.quote = null; + }else { + if(options.quote === undefined || options.quote === true){ + options.quote = Buffer.from('"', options.encoding); + }else if(typeof options.quote === 'string'){ + options.quote = Buffer.from(options.quote, options.encoding); + } + if(!isBuffer$1(options.quote)){ + throw new Error(`Invalid Option: quote must be a buffer or a string, got ${JSON.stringify(options.quote)}`); + } + } + // Normalize option `raw` + if(options.raw === undefined || options.raw === null || options.raw === false){ + options.raw = false; + }else if(options.raw !== true){ + throw new Error(`Invalid Option: raw must be true, got ${JSON.stringify(options.raw)}`); + } + // Normalize option `record_delimiter` + if(options.record_delimiter === undefined){ + options.record_delimiter = []; + }else if(typeof options.record_delimiter === 'string' || isBuffer$1(options.record_delimiter)){ + if(options.record_delimiter.length === 0){ throw new CsvError$1('CSV_INVALID_OPTION_RECORD_DELIMITER', [ 'Invalid option `record_delimiter`:', - 'value must be a string, a buffer or array of string|buffer,', + 'value must be a non empty string or buffer,', `got ${JSON.stringify(options.record_delimiter)}` ], options); } - options.record_delimiter = options.record_delimiter.map(function(rd, i){ - if(typeof rd !== 'string' && ! isBuffer$1(rd)){ - throw new CsvError$1('CSV_INVALID_OPTION_RECORD_DELIMITER', [ - 'Invalid option `record_delimiter`:', - 'value must be a string, a buffer or array of string|buffer', - `at index ${i},`, - `got ${JSON.stringify(rd)}` - ], options); - }else if(rd.length === 0){ - throw new CsvError$1('CSV_INVALID_OPTION_RECORD_DELIMITER', [ - 'Invalid option `record_delimiter`:', - 'value must be a non empty string or buffer', - `at index ${i},`, - `got ${JSON.stringify(rd)}` - ], options); - } - if(typeof rd === 'string'){ - rd = Buffer.from(rd, options.encoding); - } - return rd; - }); - // Normalize option `relax_column_count` - if(typeof options.relax_column_count === 'boolean');else if(options.relax_column_count === undefined || options.relax_column_count === null){ - options.relax_column_count = false; - }else { - throw new Error(`Invalid Option: relax_column_count must be a boolean, got ${JSON.stringify(options.relax_column_count)}`); - } - if(typeof options.relax_column_count_less === 'boolean');else if(options.relax_column_count_less === undefined || options.relax_column_count_less === null){ - options.relax_column_count_less = false; - }else { - throw new Error(`Invalid Option: relax_column_count_less must be a boolean, got ${JSON.stringify(options.relax_column_count_less)}`); - } - if(typeof options.relax_column_count_more === 'boolean');else if(options.relax_column_count_more === undefined || options.relax_column_count_more === null){ - options.relax_column_count_more = false; - }else { - throw new Error(`Invalid Option: relax_column_count_more must be a boolean, got ${JSON.stringify(options.relax_column_count_more)}`); - } - // Normalize option `relax_quotes` - if(typeof options.relax_quotes === 'boolean');else if(options.relax_quotes === undefined || options.relax_quotes === null){ - options.relax_quotes = false; - }else { - throw new Error(`Invalid Option: relax_quotes must be a boolean, got ${JSON.stringify(options.relax_quotes)}`); + options.record_delimiter = [options.record_delimiter]; + }else if(!Array.isArray(options.record_delimiter)){ + throw new CsvError$1('CSV_INVALID_OPTION_RECORD_DELIMITER', [ + 'Invalid option `record_delimiter`:', + 'value must be a string, a buffer or array of string|buffer,', + `got ${JSON.stringify(options.record_delimiter)}` + ], options); + } + options.record_delimiter = options.record_delimiter.map(function(rd, i){ + if(typeof rd !== 'string' && ! isBuffer$1(rd)){ + throw new CsvError$1('CSV_INVALID_OPTION_RECORD_DELIMITER', [ + 'Invalid option `record_delimiter`:', + 'value must be a string, a buffer or array of string|buffer', + `at index ${i},`, + `got ${JSON.stringify(rd)}` + ], options); + }else if(rd.length === 0){ + throw new CsvError$1('CSV_INVALID_OPTION_RECORD_DELIMITER', [ + 'Invalid option `record_delimiter`:', + 'value must be a non empty string or buffer', + `at index ${i},`, + `got ${JSON.stringify(rd)}` + ], options); } - // Normalize option `skip_empty_lines` - if(typeof options.skip_empty_lines === 'boolean');else if(options.skip_empty_lines === undefined || options.skip_empty_lines === null){ - options.skip_empty_lines = false; - }else { - throw new Error(`Invalid Option: skip_empty_lines must be a boolean, got ${JSON.stringify(options.skip_empty_lines)}`); + if(typeof rd === 'string'){ + rd = Buffer.from(rd, options.encoding); } - // Normalize option `skip_records_with_empty_values` - if(typeof options.skip_records_with_empty_values === 'boolean');else if(options.skip_records_with_empty_values === undefined || options.skip_records_with_empty_values === null){ - options.skip_records_with_empty_values = false; - }else { - throw new Error(`Invalid Option: skip_records_with_empty_values must be a boolean, got ${JSON.stringify(options.skip_records_with_empty_values)}`); + return rd; + }); + // Normalize option `relax_column_count` + if(typeof options.relax_column_count === 'boolean');else if(options.relax_column_count === undefined || options.relax_column_count === null){ + options.relax_column_count = false; + }else { + throw new Error(`Invalid Option: relax_column_count must be a boolean, got ${JSON.stringify(options.relax_column_count)}`); + } + if(typeof options.relax_column_count_less === 'boolean');else if(options.relax_column_count_less === undefined || options.relax_column_count_less === null){ + options.relax_column_count_less = false; + }else { + throw new Error(`Invalid Option: relax_column_count_less must be a boolean, got ${JSON.stringify(options.relax_column_count_less)}`); + } + if(typeof options.relax_column_count_more === 'boolean');else if(options.relax_column_count_more === undefined || options.relax_column_count_more === null){ + options.relax_column_count_more = false; + }else { + throw new Error(`Invalid Option: relax_column_count_more must be a boolean, got ${JSON.stringify(options.relax_column_count_more)}`); + } + // Normalize option `relax_quotes` + if(typeof options.relax_quotes === 'boolean');else if(options.relax_quotes === undefined || options.relax_quotes === null){ + options.relax_quotes = false; + }else { + throw new Error(`Invalid Option: relax_quotes must be a boolean, got ${JSON.stringify(options.relax_quotes)}`); + } + // Normalize option `skip_empty_lines` + if(typeof options.skip_empty_lines === 'boolean');else if(options.skip_empty_lines === undefined || options.skip_empty_lines === null){ + options.skip_empty_lines = false; + }else { + throw new Error(`Invalid Option: skip_empty_lines must be a boolean, got ${JSON.stringify(options.skip_empty_lines)}`); + } + // Normalize option `skip_records_with_empty_values` + if(typeof options.skip_records_with_empty_values === 'boolean');else if(options.skip_records_with_empty_values === undefined || options.skip_records_with_empty_values === null){ + options.skip_records_with_empty_values = false; + }else { + throw new Error(`Invalid Option: skip_records_with_empty_values must be a boolean, got ${JSON.stringify(options.skip_records_with_empty_values)}`); + } + // Normalize option `skip_records_with_error` + if(typeof options.skip_records_with_error === 'boolean');else if(options.skip_records_with_error === undefined || options.skip_records_with_error === null){ + options.skip_records_with_error = false; + }else { + throw new Error(`Invalid Option: skip_records_with_error must be a boolean, got ${JSON.stringify(options.skip_records_with_error)}`); + } + // Normalize option `rtrim` + if(options.rtrim === undefined || options.rtrim === null || options.rtrim === false){ + options.rtrim = false; + }else if(options.rtrim !== true){ + throw new Error(`Invalid Option: rtrim must be a boolean, got ${JSON.stringify(options.rtrim)}`); + } + // Normalize option `ltrim` + if(options.ltrim === undefined || options.ltrim === null || options.ltrim === false){ + options.ltrim = false; + }else if(options.ltrim !== true){ + throw new Error(`Invalid Option: ltrim must be a boolean, got ${JSON.stringify(options.ltrim)}`); + } + // Normalize option `trim` + if(options.trim === undefined || options.trim === null || options.trim === false){ + options.trim = false; + }else if(options.trim !== true){ + throw new Error(`Invalid Option: trim must be a boolean, got ${JSON.stringify(options.trim)}`); + } + // Normalize options `trim`, `ltrim` and `rtrim` + if(options.trim === true && opts.ltrim !== false){ + options.ltrim = true; + }else if(options.ltrim !== true){ + options.ltrim = false; + } + if(options.trim === true && opts.rtrim !== false){ + options.rtrim = true; + }else if(options.rtrim !== true){ + options.rtrim = false; + } + // Normalize option `to` + if(options.to === undefined || options.to === null){ + options.to = -1; + }else { + if(typeof options.to === 'string' && /\d+/.test(options.to)){ + options.to = parseInt(options.to); } - // Normalize option `skip_records_with_error` - if(typeof options.skip_records_with_error === 'boolean');else if(options.skip_records_with_error === undefined || options.skip_records_with_error === null){ - options.skip_records_with_error = false; - }else { - throw new Error(`Invalid Option: skip_records_with_error must be a boolean, got ${JSON.stringify(options.skip_records_with_error)}`); - } - // Normalize option `rtrim` - if(options.rtrim === undefined || options.rtrim === null || options.rtrim === false){ - options.rtrim = false; - }else if(options.rtrim !== true){ - throw new Error(`Invalid Option: rtrim must be a boolean, got ${JSON.stringify(options.rtrim)}`); - } - // Normalize option `ltrim` - if(options.ltrim === undefined || options.ltrim === null || options.ltrim === false){ - options.ltrim = false; - }else if(options.ltrim !== true){ - throw new Error(`Invalid Option: ltrim must be a boolean, got ${JSON.stringify(options.ltrim)}`); - } - // Normalize option `trim` - if(options.trim === undefined || options.trim === null || options.trim === false){ - options.trim = false; - }else if(options.trim !== true){ - throw new Error(`Invalid Option: trim must be a boolean, got ${JSON.stringify(options.trim)}`); - } - // Normalize options `trim`, `ltrim` and `rtrim` - if(options.trim === true && opts.ltrim !== false){ - options.ltrim = true; - }else if(options.ltrim !== true){ - options.ltrim = false; - } - if(options.trim === true && opts.rtrim !== false){ - options.rtrim = true; - }else if(options.rtrim !== true){ - options.rtrim = false; - } - // Normalize option `to` - if(options.to === undefined || options.to === null){ - options.to = -1; - }else { - if(typeof options.to === 'string' && /\d+/.test(options.to)){ - options.to = parseInt(options.to); - } - if(Number.isInteger(options.to)){ - if(options.to <= 0){ - throw new Error(`Invalid Option: to must be a positive integer greater than 0, got ${JSON.stringify(opts.to)}`); - } - }else { - throw new Error(`Invalid Option: to must be an integer, got ${JSON.stringify(opts.to)}`); + if(Number.isInteger(options.to)){ + if(options.to <= 0){ + throw new Error(`Invalid Option: to must be a positive integer greater than 0, got ${JSON.stringify(opts.to)}`); } - } - // Normalize option `to_line` - if(options.to_line === undefined || options.to_line === null){ - options.to_line = -1; }else { - if(typeof options.to_line === 'string' && /\d+/.test(options.to_line)){ - options.to_line = parseInt(options.to_line); - } - if(Number.isInteger(options.to_line)){ - if(options.to_line <= 0){ - throw new Error(`Invalid Option: to_line must be a positive integer greater than 0, got ${JSON.stringify(opts.to_line)}`); - } - }else { - throw new Error(`Invalid Option: to_line must be an integer, got ${JSON.stringify(opts.to_line)}`); - } + throw new Error(`Invalid Option: to must be an integer, got ${JSON.stringify(opts.to)}`); } - this.info = { - bytes: 0, - comment_lines: 0, - empty_lines: 0, - invalid_field_length: 0, - lines: 1, - records: 0 - }; - this.options = options; - this.state = { - bomSkipped: false, - bufBytesStart: 0, - castField: fnCastField, - commenting: false, - // Current error encountered by a record - error: undefined, - enabled: options.from_line === 1, - escaping: false, - escapeIsQuote: isBuffer$1(options.escape) && isBuffer$1(options.quote) && Buffer.compare(options.escape, options.quote) === 0, - // columns can be `false`, `true`, `Array` - expectedRecordLength: Array.isArray(options.columns) ? options.columns.length : undefined, - field: new ResizeableBuffer(20), - firstLineToHeaders: fnFirstLineToHeaders, - needMoreDataSize: Math.max( - // Skip if the remaining buffer smaller than comment - options.comment !== null ? options.comment.length : 0, - // Skip if the remaining buffer can be delimiter - ...options.delimiter.map((delimiter) => delimiter.length), - // Skip if the remaining buffer can be escape sequence - options.quote !== null ? options.quote.length : 0, - ), - previousBuf: undefined, - quoting: false, - stop: false, - rawBuffer: new ResizeableBuffer(100), - record: [], - recordHasError: false, - record_length: 0, - recordDelimiterMaxLength: options.record_delimiter.length === 0 ? 2 : Math.max(...options.record_delimiter.map((v) => v.length)), - trimChars: [Buffer.from(' ', options.encoding)[0], Buffer.from('\t', options.encoding)[0]], - wasQuoting: false, - wasRowDelimiter: false - }; } - // Implementation of `Transform._transform` - _transform(buf, encoding, callback){ - if(this.state.stop === true){ - return; - } - const err = this.__parse(buf, false); - if(err !== undefined){ - this.state.stop = true; - } - callback(err); - } - // Implementation of `Transform._flush` - _flush(callback){ - if(this.state.stop === true){ - return; + // Normalize option `to_line` + if(options.to_line === undefined || options.to_line === null){ + options.to_line = -1; + }else { + if(typeof options.to_line === 'string' && /\d+/.test(options.to_line)){ + options.to_line = parseInt(options.to_line); } - const err = this.__parse(undefined, true); - callback(err); - } - // Central parser implementation - __parse(nextBuf, end){ - const {bom, comment, escape, from_line, ltrim, max_record_size, quote, raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options; - let {record_delimiter} = this.options; - const {bomSkipped, previousBuf, rawBuffer, escapeIsQuote} = this.state; - let buf; - if(previousBuf === undefined){ - if(nextBuf === undefined){ - // Handle empty string - this.push(null); - return; - }else { - buf = nextBuf; + if(Number.isInteger(options.to_line)){ + if(options.to_line <= 0){ + throw new Error(`Invalid Option: to_line must be a positive integer greater than 0, got ${JSON.stringify(opts.to_line)}`); } - }else if(previousBuf !== undefined && nextBuf === undefined){ - buf = previousBuf; }else { - buf = Buffer.concat([previousBuf, nextBuf]); - } - // Handle UTF BOM - if(bomSkipped === false){ - if(bom === false){ - this.state.bomSkipped = true; - }else if(buf.length < 3){ - // No enough data - if(end === false){ - // Wait for more data - this.state.previousBuf = buf; + throw new Error(`Invalid Option: to_line must be an integer, got ${JSON.stringify(opts.to_line)}`); + } + } + return options; +}; + +const isRecordEmpty = function(record){ + return record.every((field) => field == null || field.toString && field.toString().trim() === ''); +}; + +// white space characters +// https://en.wikipedia.org/wiki/Whitespace_character +// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions/Character_Classes#Types +// \f\n\r\t\v\u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff +const tab = 9; +const nl = 10; // \n, 0x0A in hexadecimal, 10 in decimal +const np = 12; +const cr = 13; // \r, 0x0D in hexadécimal, 13 in decimal +const space = 32; +const boms = { + // Note, the following are equals: + // Buffer.from("\ufeff") + // Buffer.from([239, 187, 191]) + // Buffer.from('EFBBBF', 'hex') + 'utf8': Buffer.from([239, 187, 191]), + // Note, the following are equals: + // Buffer.from "\ufeff", 'utf16le + // Buffer.from([255, 254]) + 'utf16le': Buffer.from([255, 254]) +}; + +const transform$1 = function(original_options = {}) { + const info = { + bytes: 0, + comment_lines: 0, + empty_lines: 0, + invalid_field_length: 0, + lines: 1, + records: 0 + }; + const options = normalize_options$1(original_options); + return { + info: info, + original_options: original_options, + options: options, + state: init_state(options), + __needMoreData: function(i, bufLen, end){ + if(end) return false; + const {quote} = this.options; + const {quoting, needMoreDataSize, recordDelimiterMaxLength} = this.state; + const numOfCharLeft = bufLen - i - 1; + const requiredLength = Math.max( + needMoreDataSize, + // Skip if the remaining buffer smaller than record delimiter + recordDelimiterMaxLength, + // Skip if the remaining buffer can be record delimiter following the closing quote + // 1 is for quote.length + quoting ? (quote.length + recordDelimiterMaxLength) : 0, + ); + return numOfCharLeft < requiredLength; + }, + // Central parser implementation + parse: function(nextBuf, end, push, close){ + const {bom, comment, escape, from_line, ltrim, max_record_size, quote, raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options; + let {record_delimiter} = this.options; + const {bomSkipped, previousBuf, rawBuffer, escapeIsQuote} = this.state; + let buf; + if(previousBuf === undefined){ + if(nextBuf === undefined){ + // Handle empty string + close(); return; + }else { + buf = nextBuf; } + }else if(previousBuf !== undefined && nextBuf === undefined){ + buf = previousBuf; }else { - for(const encoding in boms){ - if(boms[encoding].compare(buf, 0, boms[encoding].length) === 0){ - // Skip BOM - const bomLength = boms[encoding].length; - this.state.bufBytesStart += bomLength; - buf = buf.slice(bomLength); - // Renormalize original options with the new encoding - this.__normalizeOptions({...this.__originalOptions, encoding: encoding}); - break; + buf = Buffer.concat([previousBuf, nextBuf]); + } + // Handle UTF BOM + if(bomSkipped === false){ + if(bom === false){ + this.state.bomSkipped = true; + }else if(buf.length < 3){ + // No enough data + if(end === false){ + // Wait for more data + this.state.previousBuf = buf; + return; } + }else { + for(const encoding in boms){ + if(boms[encoding].compare(buf, 0, boms[encoding].length) === 0){ + // Skip BOM + const bomLength = boms[encoding].length; + this.state.bufBytesStart += bomLength; + buf = buf.slice(bomLength); + // Renormalize original options with the new encoding + this.options = normalize_options$1({...this.original_options, encoding: encoding}); + break; + } + } + this.state.bomSkipped = true; } - this.state.bomSkipped = true; - } - } - const bufLen = buf.length; - let pos; - for(pos = 0; pos < bufLen; pos++){ - // Ensure we get enough space to look ahead - // There should be a way to move this out of the loop - if(this.__needMoreData(pos, bufLen, end)){ - break; - } - if(this.state.wasRowDelimiter === true){ - this.info.lines++; - this.state.wasRowDelimiter = false; - } - if(to_line !== -1 && this.info.lines > to_line){ - this.state.stop = true; - this.push(null); - return; } - // Auto discovery of record_delimiter, unix, mac and windows supported - if(this.state.quoting === false && record_delimiter.length === 0){ - const record_delimiterCount = this.__autoDiscoverRecordDelimiter(buf, pos); - if(record_delimiterCount){ - record_delimiter = this.options.record_delimiter; + const bufLen = buf.length; + let pos; + for(pos = 0; pos < bufLen; pos++){ + // Ensure we get enough space to look ahead + // There should be a way to move this out of the loop + if(this.__needMoreData(pos, bufLen, end)){ + break; } - } - const chr = buf[pos]; - if(raw === true){ - rawBuffer.append(chr); - } - if((chr === cr || chr === nl) && this.state.wasRowDelimiter === false){ - this.state.wasRowDelimiter = true; - } - // Previous char was a valid escape char - // treat the current char as a regular char - if(this.state.escaping === true){ - this.state.escaping = false; - }else { - // Escape is only active inside quoted fields - // We are quoting, the char is an escape chr and there is a chr to escape - // if(escape !== null && this.state.quoting === true && chr === escape && pos + 1 < bufLen){ - if(escape !== null && this.state.quoting === true && this.__isEscape(buf, pos, chr) && pos + escape.length < bufLen){ - if(escapeIsQuote){ - if(this.__isQuote(buf, pos+escape.length)){ - this.state.escaping = true; - pos += escape.length - 1; - continue; - } - }else { - this.state.escaping = true; - pos += escape.length - 1; - continue; + if(this.state.wasRowDelimiter === true){ + this.info.lines++; + this.state.wasRowDelimiter = false; + } + if(to_line !== -1 && this.info.lines > to_line){ + this.state.stop = true; + close(); + return; + } + // Auto discovery of record_delimiter, unix, mac and windows supported + if(this.state.quoting === false && record_delimiter.length === 0){ + const record_delimiterCount = this.__autoDiscoverRecordDelimiter(buf, pos); + if(record_delimiterCount){ + record_delimiter = this.options.record_delimiter; } } - // Not currently escaping and chr is a quote - // TODO: need to compare bytes instead of single char - if(this.state.commenting === false && this.__isQuote(buf, pos)){ - if(this.state.quoting === true){ - const nextChr = buf[pos+quote.length]; - const isNextChrTrimable = rtrim && this.__isCharTrimable(nextChr); - const isNextChrComment = comment !== null && this.__compareBytes(comment, buf, pos+quote.length, nextChr); - const isNextChrDelimiter = this.__isDelimiter(buf, pos+quote.length, nextChr); - const isNextChrRecordDelimiter = record_delimiter.length === 0 ? this.__autoDiscoverRecordDelimiter(buf, pos+quote.length) : this.__isRecordDelimiter(nextChr, buf, pos+quote.length); - // Escape a quote - // Treat next char as a regular character - if(escape !== null && this.__isEscape(buf, pos, chr) && this.__isQuote(buf, pos + escape.length)){ + const chr = buf[pos]; + if(raw === true){ + rawBuffer.append(chr); + } + if((chr === cr || chr === nl) && this.state.wasRowDelimiter === false){ + this.state.wasRowDelimiter = true; + } + // Previous char was a valid escape char + // treat the current char as a regular char + if(this.state.escaping === true){ + this.state.escaping = false; + }else { + // Escape is only active inside quoted fields + // We are quoting, the char is an escape chr and there is a chr to escape + // if(escape !== null && this.state.quoting === true && chr === escape && pos + 1 < bufLen){ + if(escape !== null && this.state.quoting === true && this.__isEscape(buf, pos, chr) && pos + escape.length < bufLen){ + if(escapeIsQuote){ + if(this.__isQuote(buf, pos+escape.length)){ + this.state.escaping = true; + pos += escape.length - 1; + continue; + } + }else { + this.state.escaping = true; pos += escape.length - 1; - }else if(!nextChr || isNextChrDelimiter || isNextChrRecordDelimiter || isNextChrComment || isNextChrTrimable){ - this.state.quoting = false; - this.state.wasQuoting = true; - pos += quote.length - 1; continue; - }else if(relax_quotes === false){ - const err = this.__error( - new CsvError$1('CSV_INVALID_CLOSING_QUOTE', [ - 'Invalid Closing Quote:', - `got "${String.fromCharCode(nextChr)}"`, - `at line ${this.info.lines}`, - 'instead of delimiter, record delimiter, trimable character', - '(if activated) or comment', - ], this.options, this.__infoField()) - ); - if(err !== undefined) return err; - }else { - this.state.quoting = false; - this.state.wasQuoting = true; - this.state.field.prepend(quote); - pos += quote.length - 1; } - }else { - if(this.state.field.length !== 0){ - // In relax_quotes mode, treat opening quote preceded by chrs as regular - if(relax_quotes === false){ + } + // Not currently escaping and chr is a quote + // TODO: need to compare bytes instead of single char + if(this.state.commenting === false && this.__isQuote(buf, pos)){ + if(this.state.quoting === true){ + const nextChr = buf[pos+quote.length]; + const isNextChrTrimable = rtrim && this.__isCharTrimable(nextChr); + const isNextChrComment = comment !== null && this.__compareBytes(comment, buf, pos+quote.length, nextChr); + const isNextChrDelimiter = this.__isDelimiter(buf, pos+quote.length, nextChr); + const isNextChrRecordDelimiter = record_delimiter.length === 0 ? this.__autoDiscoverRecordDelimiter(buf, pos+quote.length) : this.__isRecordDelimiter(nextChr, buf, pos+quote.length); + // Escape a quote + // Treat next char as a regular character + if(escape !== null && this.__isEscape(buf, pos, chr) && this.__isQuote(buf, pos + escape.length)){ + pos += escape.length - 1; + }else if(!nextChr || isNextChrDelimiter || isNextChrRecordDelimiter || isNextChrComment || isNextChrTrimable){ + this.state.quoting = false; + this.state.wasQuoting = true; + pos += quote.length - 1; + continue; + }else if(relax_quotes === false){ const err = this.__error( - new CsvError$1('INVALID_OPENING_QUOTE', [ - 'Invalid Opening Quote:', - `a quote is found inside a field at line ${this.info.lines}`, - ], this.options, this.__infoField(), { - field: this.state.field, - }) + new CsvError$1('CSV_INVALID_CLOSING_QUOTE', [ + 'Invalid Closing Quote:', + `got "${String.fromCharCode(nextChr)}"`, + `at line ${this.info.lines}`, + 'instead of delimiter, record delimiter, trimable character', + '(if activated) or comment', + ], this.options, this.__infoField()) ); if(err !== undefined) return err; + }else { + this.state.quoting = false; + this.state.wasQuoting = true; + this.state.field.prepend(quote); + pos += quote.length - 1; } }else { - this.state.quoting = true; - pos += quote.length - 1; - continue; - } - } - } - if(this.state.quoting === false){ - const recordDelimiterLength = this.__isRecordDelimiter(chr, buf, pos); - if(recordDelimiterLength !== 0){ - // Do not emit comments which take a full line - const skipCommentLine = this.state.commenting && (this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0); - if(skipCommentLine){ - this.info.comment_lines++; - // Skip full comment line - }else { - // Activate records emition if above from_line - if(this.state.enabled === false && this.info.lines + (this.state.wasRowDelimiter === true ? 1: 0) >= from_line){ - this.state.enabled = true; - this.__resetField(); - this.__resetRecord(); - pos += recordDelimiterLength - 1; + if(this.state.field.length !== 0){ + // In relax_quotes mode, treat opening quote preceded by chrs as regular + if(relax_quotes === false){ + const err = this.__error( + new CsvError$1('INVALID_OPENING_QUOTE', [ + 'Invalid Opening Quote:', + `a quote is found inside a field at line ${this.info.lines}`, + ], this.options, this.__infoField(), { + field: this.state.field, + }) + ); + if(err !== undefined) return err; + } + }else { + this.state.quoting = true; + pos += quote.length - 1; continue; } - // Skip if line is empty and skip_empty_lines activated - if(skip_empty_lines === true && this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0){ - this.info.empty_lines++; - pos += recordDelimiterLength - 1; - continue; + } + } + if(this.state.quoting === false){ + const recordDelimiterLength = this.__isRecordDelimiter(chr, buf, pos); + if(recordDelimiterLength !== 0){ + // Do not emit comments which take a full line + const skipCommentLine = this.state.commenting && (this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0); + if(skipCommentLine){ + this.info.comment_lines++; + // Skip full comment line + }else { + // Activate records emition if above from_line + if(this.state.enabled === false && this.info.lines + (this.state.wasRowDelimiter === true ? 1: 0) >= from_line){ + this.state.enabled = true; + this.__resetField(); + this.__resetRecord(); + pos += recordDelimiterLength - 1; + continue; + } + // Skip if line is empty and skip_empty_lines activated + if(skip_empty_lines === true && this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0){ + this.info.empty_lines++; + pos += recordDelimiterLength - 1; + continue; + } + this.info.bytes = this.state.bufBytesStart + pos; + const errField = this.__onField(); + if(errField !== undefined) return errField; + this.info.bytes = this.state.bufBytesStart + pos + recordDelimiterLength; + const errRecord = this.__onRecord(push); + if(errRecord !== undefined) return errRecord; + if(to !== -1 && this.info.records >= to){ + this.state.stop = true; + close(); + return; + } } + this.state.commenting = false; + pos += recordDelimiterLength - 1; + continue; + } + if(this.state.commenting){ + continue; + } + const commentCount = comment === null ? 0 : this.__compareBytes(comment, buf, pos, chr); + if(commentCount !== 0){ + this.state.commenting = true; + continue; + } + const delimiterLength = this.__isDelimiter(buf, pos, chr); + if(delimiterLength !== 0){ this.info.bytes = this.state.bufBytesStart + pos; const errField = this.__onField(); if(errField !== undefined) return errField; - this.info.bytes = this.state.bufBytesStart + pos + recordDelimiterLength; - const errRecord = this.__onRecord(); - if(errRecord !== undefined) return errRecord; - if(to !== -1 && this.info.records >= to){ - this.state.stop = true; - this.push(null); - return; - } + pos += delimiterLength - 1; + continue; } - this.state.commenting = false; - pos += recordDelimiterLength - 1; - continue; - } - if(this.state.commenting){ - continue; - } - const commentCount = comment === null ? 0 : this.__compareBytes(comment, buf, pos, chr); - if(commentCount !== 0){ - this.state.commenting = true; - continue; } - const delimiterLength = this.__isDelimiter(buf, pos, chr); - if(delimiterLength !== 0){ - this.info.bytes = this.state.bufBytesStart + pos; - const errField = this.__onField(); - if(errField !== undefined) return errField; - pos += delimiterLength - 1; - continue; + } + if(this.state.commenting === false){ + if(max_record_size !== 0 && this.state.record_length + this.state.field.length > max_record_size){ + const err = this.__error( + new CsvError$1('CSV_MAX_RECORD_SIZE', [ + 'Max Record Size:', + 'record exceed the maximum number of tolerated bytes', + `of ${max_record_size}`, + `at line ${this.info.lines}`, + ], this.options, this.__infoField()) + ); + if(err !== undefined) return err; } } - } - if(this.state.commenting === false){ - if(max_record_size !== 0 && this.state.record_length + this.state.field.length > max_record_size){ + const lappend = ltrim === false || this.state.quoting === true || this.state.field.length !== 0 || !this.__isCharTrimable(chr); + // rtrim in non quoting is handle in __onField + const rappend = rtrim === false || this.state.wasQuoting === false; + if(lappend === true && rappend === true){ + this.state.field.append(chr); + }else if(rtrim === true && !this.__isCharTrimable(chr)){ const err = this.__error( - new CsvError$1('CSV_MAX_RECORD_SIZE', [ - 'Max Record Size:', - 'record exceed the maximum number of tolerated bytes', - `of ${max_record_size}`, + new CsvError$1('CSV_NON_TRIMABLE_CHAR_AFTER_CLOSING_QUOTE', [ + 'Invalid Closing Quote:', + 'found non trimable byte after quote', `at line ${this.info.lines}`, ], this.options, this.__infoField()) ); if(err !== undefined) return err; } } - const lappend = ltrim === false || this.state.quoting === true || this.state.field.length !== 0 || !this.__isCharTrimable(chr); - // rtrim in non quoting is handle in __onField - const rappend = rtrim === false || this.state.wasQuoting === false; - if(lappend === true && rappend === true){ - this.state.field.append(chr); - }else if(rtrim === true && !this.__isCharTrimable(chr)){ - const err = this.__error( - new CsvError$1('CSV_NON_TRIMABLE_CHAR_AFTER_CLOSING_QUOTE', [ - 'Invalid Closing Quote:', - 'found non trimable byte after quote', - `at line ${this.info.lines}`, - ], this.options, this.__infoField()) - ); - if(err !== undefined) return err; - } - } - if(end === true){ - // Ensure we are not ending in a quoting state - if(this.state.quoting === true){ - const err = this.__error( - new CsvError$1('CSV_QUOTE_NOT_CLOSED', [ - 'Quote Not Closed:', - `the parsing is finished with an opening quote at line ${this.info.lines}`, - ], this.options, this.__infoField()) - ); - if(err !== undefined) return err; + if(end === true){ + // Ensure we are not ending in a quoting state + if(this.state.quoting === true){ + const err = this.__error( + new CsvError$1('CSV_QUOTE_NOT_CLOSED', [ + 'Quote Not Closed:', + `the parsing is finished with an opening quote at line ${this.info.lines}`, + ], this.options, this.__infoField()) + ); + if(err !== undefined) return err; + }else { + // Skip last line if it has no characters + if(this.state.wasQuoting === true || this.state.record.length !== 0 || this.state.field.length !== 0){ + this.info.bytes = this.state.bufBytesStart + pos; + const errField = this.__onField(); + if(errField !== undefined) return errField; + const errRecord = this.__onRecord(push); + if(errRecord !== undefined) return errRecord; + }else if(this.state.wasRowDelimiter === true){ + this.info.empty_lines++; + }else if(this.state.commenting === true){ + this.info.comment_lines++; + } + } }else { - // Skip last line if it has no characters - if(this.state.wasQuoting === true || this.state.record.length !== 0 || this.state.field.length !== 0){ - this.info.bytes = this.state.bufBytesStart + pos; - const errField = this.__onField(); - if(errField !== undefined) return errField; - const errRecord = this.__onRecord(); - if(errRecord !== undefined) return errRecord; - }else if(this.state.wasRowDelimiter === true){ - this.info.empty_lines++; - }else if(this.state.commenting === true){ - this.info.comment_lines++; + this.state.bufBytesStart += pos; + this.state.previousBuf = buf.slice(pos); + } + if(this.state.wasRowDelimiter === true){ + this.info.lines++; + this.state.wasRowDelimiter = false; + } + }, + __onRecord: function(push){ + const {columns, group_columns_by_name, encoding, info, from, relax_column_count, relax_column_count_less, relax_column_count_more, raw, skip_records_with_empty_values} = this.options; + const {enabled, record} = this.state; + if(enabled === false){ + return this.__resetRecord(); + } + // Convert the first line into column names + const recordLength = record.length; + if(columns === true){ + if(skip_records_with_empty_values === true && isRecordEmpty(record)){ + this.__resetRecord(); + return; + } + return this.__firstLineToColumns(record); + } + if(columns === false && this.info.records === 0){ + this.state.expectedRecordLength = recordLength; + } + if(recordLength !== this.state.expectedRecordLength){ + const err = columns === false ? + new CsvError$1('CSV_RECORD_INCONSISTENT_FIELDS_LENGTH', [ + 'Invalid Record Length:', + `expect ${this.state.expectedRecordLength},`, + `got ${recordLength} on line ${this.info.lines}`, + ], this.options, this.__infoField(), { + record: record, + }) + : + new CsvError$1('CSV_RECORD_INCONSISTENT_COLUMNS', [ + 'Invalid Record Length:', + `columns length is ${columns.length},`, // rename columns + `got ${recordLength} on line ${this.info.lines}`, + ], this.options, this.__infoField(), { + record: record, + }); + if(relax_column_count === true || + (relax_column_count_less === true && recordLength < this.state.expectedRecordLength) || + (relax_column_count_more === true && recordLength > this.state.expectedRecordLength)){ + this.info.invalid_field_length++; + this.state.error = err; + // Error is undefined with skip_records_with_error + }else { + const finalErr = this.__error(err); + if(finalErr) return finalErr; } } - }else { - this.state.bufBytesStart += pos; - this.state.previousBuf = buf.slice(pos); - } - if(this.state.wasRowDelimiter === true){ - this.info.lines++; - this.state.wasRowDelimiter = false; - } - } - __onRecord(){ - const {columns, group_columns_by_name, encoding, info, from, relax_column_count, relax_column_count_less, relax_column_count_more, raw, skip_records_with_empty_values} = this.options; - const {enabled, record} = this.state; - if(enabled === false){ - return this.__resetRecord(); - } - // Convert the first line into column names - const recordLength = record.length; - if(columns === true){ if(skip_records_with_empty_values === true && isRecordEmpty(record)){ this.__resetRecord(); return; } - return this.__firstLineToColumns(record); - } - if(columns === false && this.info.records === 0){ - this.state.expectedRecordLength = recordLength; - } - if(recordLength !== this.state.expectedRecordLength){ - const err = columns === false ? - new CsvError$1('CSV_RECORD_INCONSISTENT_FIELDS_LENGTH', [ - 'Invalid Record Length:', - `expect ${this.state.expectedRecordLength},`, - `got ${recordLength} on line ${this.info.lines}`, - ], this.options, this.__infoField(), { - record: record, - }) - : - new CsvError$1('CSV_RECORD_INCONSISTENT_COLUMNS', [ - 'Invalid Record Length:', - `columns length is ${columns.length},`, // rename columns - `got ${recordLength} on line ${this.info.lines}`, - ], this.options, this.__infoField(), { - record: record, - }); - if(relax_column_count === true || - (relax_column_count_less === true && recordLength < this.state.expectedRecordLength) || - (relax_column_count_more === true && recordLength > this.state.expectedRecordLength)){ - this.info.invalid_field_length++; - this.state.error = err; - // Error is undefined with skip_records_with_error - }else { - const finalErr = this.__error(err); - if(finalErr) return finalErr; + if(this.state.recordHasError === true){ + this.__resetRecord(); + this.state.recordHasError = false; + return; } - } - if(skip_records_with_empty_values === true && isRecordEmpty(record)){ - this.__resetRecord(); - return; - } - if(this.state.recordHasError === true){ - this.__resetRecord(); - this.state.recordHasError = false; - return; - } - this.info.records++; - if(from === 1 || this.info.records >= from){ - const {objname} = this.options; - // With columns, records are object - if(columns !== false){ - const obj = {}; - // Transform record array to an object - for(let i = 0, l = record.length; i < l; i++){ - if(columns[i] === undefined || columns[i].disabled) continue; - // Turn duplicate columns into an array - if (group_columns_by_name === true && obj[columns[i].name] !== undefined) { - if (Array.isArray(obj[columns[i].name])) { - obj[columns[i].name] = obj[columns[i].name].concat(record[i]); + this.info.records++; + if(from === 1 || this.info.records >= from){ + const {objname} = this.options; + // With columns, records are object + if(columns !== false){ + const obj = {}; + // Transform record array to an object + for(let i = 0, l = record.length; i < l; i++){ + if(columns[i] === undefined || columns[i].disabled) continue; + // Turn duplicate columns into an array + if (group_columns_by_name === true && obj[columns[i].name] !== undefined) { + if (Array.isArray(obj[columns[i].name])) { + obj[columns[i].name] = obj[columns[i].name].concat(record[i]); + } else { + obj[columns[i].name] = [obj[columns[i].name], record[i]]; + } } else { - obj[columns[i].name] = [obj[columns[i].name], record[i]]; + obj[columns[i].name] = record[i]; } - } else { - obj[columns[i].name] = record[i]; } - } - // Without objname (default) - if(raw === true || info === true){ - const extRecord = Object.assign( - {record: obj}, - (raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}), - (info === true ? {info: this.__infoRecord()}: {}) - ); - const err = this.__push( - objname === undefined ? extRecord : [obj[objname], extRecord] - ); - if(err){ - return err; - } - }else { - const err = this.__push( - objname === undefined ? obj : [obj[objname], obj] - ); - if(err){ - return err; - } - } - // Without columns, records are array - }else { - if(raw === true || info === true){ - const extRecord = Object.assign( - {record: record}, - raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}, - info === true ? {info: this.__infoRecord()}: {} - ); - const err = this.__push( - objname === undefined ? extRecord : [record[objname], extRecord] - ); - if(err){ - return err; + // Without objname (default) + if(raw === true || info === true){ + const extRecord = Object.assign( + {record: obj}, + (raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}), + (info === true ? {info: this.__infoRecord()}: {}) + ); + const err = this.__push( + objname === undefined ? extRecord : [obj[objname], extRecord] + , push); + if(err){ + return err; + } + }else { + const err = this.__push( + objname === undefined ? obj : [obj[objname], obj] + , push); + if(err){ + return err; + } } + // Without columns, records are array }else { - const err = this.__push( - objname === undefined ? record : [record[objname], record] - ); - if(err){ - return err; + if(raw === true || info === true){ + const extRecord = Object.assign( + {record: record}, + raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}, + info === true ? {info: this.__infoRecord()}: {} + ); + const err = this.__push( + objname === undefined ? extRecord : [record[objname], extRecord] + , push); + if(err){ + return err; + } + }else { + const err = this.__push( + objname === undefined ? record : [record[objname], record] + , push); + if(err){ + return err; + } } } } - } - this.__resetRecord(); - } - __firstLineToColumns(record){ - const {firstLineToHeaders} = this.state; - try{ - const headers = firstLineToHeaders === undefined ? record : firstLineToHeaders.call(null, record); - if(!Array.isArray(headers)){ - return this.__error( - new CsvError$1('CSV_INVALID_COLUMN_MAPPING', [ - 'Invalid Column Mapping:', - 'expect an array from column function,', - `got ${JSON.stringify(headers)}` - ], this.options, this.__infoField(), { - headers: headers, - }) - ); - } - const normalizedHeaders = normalizeColumnsArray(headers); - this.state.expectedRecordLength = normalizedHeaders.length; - this.options.columns = normalizedHeaders; this.__resetRecord(); - return; - }catch(err){ - return err; - } - } - __resetRecord(){ - if(this.options.raw === true){ - this.state.rawBuffer.reset(); - } - this.state.error = undefined; - this.state.record = []; - this.state.record_length = 0; - } - __onField(){ - const {cast, encoding, rtrim, max_record_size} = this.options; - const {enabled, wasQuoting} = this.state; - // Short circuit for the from_line options - if(enabled === false){ - return this.__resetField(); - } - let field = this.state.field.toString(encoding); - if(rtrim === true && wasQuoting === false){ - field = field.trimRight(); - } - if(cast === true){ - const [err, f] = this.__cast(field); - if(err !== undefined) return err; - field = f; - } - this.state.record.push(field); - // Increment record length if record size must not exceed a limit - if(max_record_size !== 0 && typeof field === 'string'){ - this.state.record_length += field.length; - } - this.__resetField(); - } - __resetField(){ - this.state.field.reset(); - this.state.wasQuoting = false; - } - __push(record){ - const {on_record} = this.options; - if(on_record !== undefined){ - const info = this.__infoRecord(); + }, + __firstLineToColumns: function(record){ + const {firstLineToHeaders} = this.state; try{ - record = on_record.call(null, record, info); + const headers = firstLineToHeaders === undefined ? record : firstLineToHeaders.call(null, record); + if(!Array.isArray(headers)){ + return this.__error( + new CsvError$1('CSV_INVALID_COLUMN_MAPPING', [ + 'Invalid Column Mapping:', + 'expect an array from column function,', + `got ${JSON.stringify(headers)}` + ], this.options, this.__infoField(), { + headers: headers, + }) + ); + } + const normalizedHeaders = normalize_columns_array(headers); + this.state.expectedRecordLength = normalizedHeaders.length; + this.options.columns = normalizedHeaders; + this.__resetRecord(); + return; }catch(err){ return err; } - if(record === undefined || record === null){ return; } - } - this.push(record); - } - // Return a tuple with the error and the casted value - __cast(field){ - const {columns, relax_column_count} = this.options; - const isColumns = Array.isArray(columns); - // Dont loose time calling cast - // because the final record is an object - // and this field can't be associated to a key present in columns - if(isColumns === true && relax_column_count && this.options.columns.length <= this.state.record.length){ - return [undefined, undefined]; - } - if(this.state.castField !== null){ - try{ + }, + __resetRecord: function(){ + if(this.options.raw === true){ + this.state.rawBuffer.reset(); + } + this.state.error = undefined; + this.state.record = []; + this.state.record_length = 0; + }, + __onField: function(){ + const {cast, encoding, rtrim, max_record_size} = this.options; + const {enabled, wasQuoting} = this.state; + // Short circuit for the from_line options + if(enabled === false){ + return this.__resetField(); + } + let field = this.state.field.toString(encoding); + if(rtrim === true && wasQuoting === false){ + field = field.trimRight(); + } + if(cast === true){ + const [err, f] = this.__cast(field); + if(err !== undefined) return err; + field = f; + } + this.state.record.push(field); + // Increment record length if record size must not exceed a limit + if(max_record_size !== 0 && typeof field === 'string'){ + this.state.record_length += field.length; + } + this.__resetField(); + }, + __resetField: function(){ + this.state.field.reset(); + this.state.wasQuoting = false; + }, + __push: function(record, push){ + const {on_record} = this.options; + if(on_record !== undefined){ + const info = this.__infoRecord(); + try{ + record = on_record.call(null, record, info); + }catch(err){ + return err; + } + if(record === undefined || record === null){ return; } + } + push(record); + }, + // Return a tuple with the error and the casted value + __cast: function(field){ + const {columns, relax_column_count} = this.options; + const isColumns = Array.isArray(columns); + // Dont loose time calling cast + // because the final record is an object + // and this field can't be associated to a key present in columns + if(isColumns === true && relax_column_count && this.options.columns.length <= this.state.record.length){ + return [undefined, undefined]; + } + if(this.state.castField !== null){ + try{ + const info = this.__infoField(); + return [undefined, this.state.castField.call(null, field, info)]; + }catch(err){ + return [err]; + } + } + if(this.__isFloat(field)){ + return [undefined, parseFloat(field)]; + }else if(this.options.cast_date !== false){ const info = this.__infoField(); - return [undefined, this.state.castField.call(null, field, info)]; - }catch(err){ - return [err]; + return [undefined, this.options.cast_date.call(null, field, info)]; } - } - if(this.__isFloat(field)){ - return [undefined, parseFloat(field)]; - }else if(this.options.cast_date !== false){ - const info = this.__infoField(); - return [undefined, this.options.cast_date.call(null, field, info)]; - } - return [undefined, field]; - } - // Helper to test if a character is a space or a line delimiter - __isCharTrimable(chr){ - return chr === space || chr === tab || chr === cr || chr === nl || chr === np; - } - // Keep it in case we implement the `cast_int` option - // __isInt(value){ - // // return Number.isInteger(parseInt(value)) - // // return !isNaN( parseInt( obj ) ); - // return /^(\-|\+)?[1-9][0-9]*$/.test(value) - // } - __isFloat(value){ - return (value - parseFloat(value) + 1) >= 0; // Borrowed from jquery - } - __compareBytes(sourceBuf, targetBuf, targetPos, firstByte){ - if(sourceBuf[0] !== firstByte) return 0; - const sourceLength = sourceBuf.length; - for(let i = 1; i < sourceLength; i++){ - if(sourceBuf[i] !== targetBuf[targetPos+i]) return 0; - } - return sourceLength; - } - __needMoreData(i, bufLen, end){ - if(end) return false; - const {quote} = this.options; - const {quoting, needMoreDataSize, recordDelimiterMaxLength} = this.state; - const numOfCharLeft = bufLen - i - 1; - const requiredLength = Math.max( - needMoreDataSize, - // Skip if the remaining buffer smaller than record delimiter - recordDelimiterMaxLength, - // Skip if the remaining buffer can be record delimiter following the closing quote - // 1 is for quote.length - quoting ? (quote.length + recordDelimiterMaxLength) : 0, - ); - return numOfCharLeft < requiredLength; - } - __isDelimiter(buf, pos, chr){ - const {delimiter, ignore_last_delimiters} = this.options; - if(ignore_last_delimiters === true && this.state.record.length === this.options.columns.length - 1){ - return 0; - }else if(ignore_last_delimiters !== false && typeof ignore_last_delimiters === 'number' && this.state.record.length === ignore_last_delimiters - 1){ - return 0; - } - loop1: for(let i = 0; i < delimiter.length; i++){ - const del = delimiter[i]; - if(del[0] === chr){ - for(let j = 1; j < del.length; j++){ - if(del[j] !== buf[pos+j]) continue loop1; + return [undefined, field]; + }, + // Helper to test if a character is a space or a line delimiter + __isCharTrimable: function(chr){ + return chr === space || chr === tab || chr === cr || chr === nl || chr === np; + }, + // Keep it in case we implement the `cast_int` option + // __isInt(value){ + // // return Number.isInteger(parseInt(value)) + // // return !isNaN( parseInt( obj ) ); + // return /^(\-|\+)?[1-9][0-9]*$/.test(value) + // } + __isFloat: function(value){ + return (value - parseFloat(value) + 1) >= 0; // Borrowed from jquery + }, + __compareBytes: function(sourceBuf, targetBuf, targetPos, firstByte){ + if(sourceBuf[0] !== firstByte) return 0; + const sourceLength = sourceBuf.length; + for(let i = 1; i < sourceLength; i++){ + if(sourceBuf[i] !== targetBuf[targetPos+i]) return 0; + } + return sourceLength; + }, + __isDelimiter: function(buf, pos, chr){ + const {delimiter, ignore_last_delimiters} = this.options; + if(ignore_last_delimiters === true && this.state.record.length === this.options.columns.length - 1){ + return 0; + }else if(ignore_last_delimiters !== false && typeof ignore_last_delimiters === 'number' && this.state.record.length === ignore_last_delimiters - 1){ + return 0; + } + loop1: for(let i = 0; i < delimiter.length; i++){ + const del = delimiter[i]; + if(del[0] === chr){ + for(let j = 1; j < del.length; j++){ + if(del[j] !== buf[pos+j]) continue loop1; + } + return del.length; } - return del.length; } - } - return 0; - } - __isRecordDelimiter(chr, buf, pos){ - const {record_delimiter} = this.options; - const recordDelimiterLength = record_delimiter.length; - loop1: for(let i = 0; i < recordDelimiterLength; i++){ - const rd = record_delimiter[i]; - const rdLength = rd.length; - if(rd[0] !== chr){ - continue; + return 0; + }, + __isRecordDelimiter: function(chr, buf, pos){ + const {record_delimiter} = this.options; + const recordDelimiterLength = record_delimiter.length; + loop1: for(let i = 0; i < recordDelimiterLength; i++){ + const rd = record_delimiter[i]; + const rdLength = rd.length; + if(rd[0] !== chr){ + continue; + } + for(let j = 1; j < rdLength; j++){ + if(rd[j] !== buf[pos+j]){ + continue loop1; + } + } + return rd.length; } - for(let j = 1; j < rdLength; j++){ - if(rd[j] !== buf[pos+j]){ - continue loop1; + return 0; + }, + __isEscape: function(buf, pos, chr){ + const {escape} = this.options; + if(escape === null) return false; + const l = escape.length; + if(escape[0] === chr){ + for(let i = 0; i < l; i++){ + if(escape[i] !== buf[pos+i]){ + return false; + } } + return true; } - return rd.length; - } - return 0; - } - __isEscape(buf, pos, chr){ - const {escape} = this.options; - if(escape === null) return false; - const l = escape.length; - if(escape[0] === chr){ + return false; + }, + __isQuote: function(buf, pos){ + const {quote} = this.options; + if(quote === null) return false; + const l = quote.length; for(let i = 0; i < l; i++){ - if(escape[i] !== buf[pos+i]){ + if(quote[i] !== buf[pos+i]){ return false; } } return true; - } - return false; - } - __isQuote(buf, pos){ - const {quote} = this.options; - if(quote === null) return false; - const l = quote.length; - for(let i = 0; i < l; i++){ - if(quote[i] !== buf[pos+i]){ - return false; - } - } - return true; - } - __autoDiscoverRecordDelimiter(buf, pos){ - const {encoding} = this.options; - const chr = buf[pos]; - if(chr === cr){ - if(buf[pos+1] === nl){ - this.options.record_delimiter.push(Buffer.from('\r\n', encoding)); - this.state.recordDelimiterMaxLength = 2; - return 2; - }else { - this.options.record_delimiter.push(Buffer.from('\r', encoding)); + }, + __autoDiscoverRecordDelimiter: function(buf, pos){ + const {encoding} = this.options; + const chr = buf[pos]; + if(chr === cr){ + if(buf[pos+1] === nl){ + this.options.record_delimiter.push(Buffer.from('\r\n', encoding)); + this.state.recordDelimiterMaxLength = 2; + return 2; + }else { + this.options.record_delimiter.push(Buffer.from('\r', encoding)); + this.state.recordDelimiterMaxLength = 1; + return 1; + } + }else if(chr === nl){ + this.options.record_delimiter.push(Buffer.from('\n', encoding)); this.state.recordDelimiterMaxLength = 1; return 1; } - }else if(chr === nl){ - this.options.record_delimiter.push(Buffer.from('\n', encoding)); - this.state.recordDelimiterMaxLength = 1; - return 1; - } - return 0; - } - __error(msg){ - const {encoding, raw, skip_records_with_error} = this.options; - const err = typeof msg === 'string' ? new Error(msg) : msg; - if(skip_records_with_error){ - this.state.recordHasError = true; - this.emit('skip', err, raw ? this.state.rawBuffer.toString(encoding) : undefined); - return undefined; - }else { - return err; + return 0; + }, + __error: function(msg){ + const {encoding, raw, skip_records_with_error} = this.options; + const err = typeof msg === 'string' ? new Error(msg) : msg; + if(skip_records_with_error){ + this.state.recordHasError = true; + if(this.options.on_skip !== undefined){ + this.options.on_skip(err, raw ? this.state.rawBuffer.toString(encoding) : undefined); + } + // this.emit('skip', err, raw ? this.state.rawBuffer.toString(encoding) : undefined); + return undefined; + }else { + return err; + } + }, + __infoDataSet: function(){ + return { + ...this.info, + columns: this.options.columns + }; + }, + __infoRecord: function(){ + const {columns, raw, encoding} = this.options; + return { + ...this.__infoDataSet(), + error: this.state.error, + header: columns === true, + index: this.state.record.length, + raw: raw ? this.state.rawBuffer.toString(encoding) : undefined + }; + }, + __infoField: function(){ + const {columns} = this.options; + const isColumns = Array.isArray(columns); + return { + ...this.__infoRecord(), + column: isColumns === true ? + (columns.length > this.state.record.length ? + columns[this.state.record.length].name : + null + ) : + this.state.record.length, + quoting: this.state.wasQuoting, + }; } - } - __infoDataSet(){ - return { - ...this.info, - columns: this.options.columns - }; - } - __infoRecord(){ - const {columns, raw, encoding} = this.options; - return { - ...this.__infoDataSet(), - error: this.state.error, - header: columns === true, - index: this.state.record.length, - raw: raw ? this.state.rawBuffer.toString(encoding) : undefined - }; - } - __infoField(){ - const {columns} = this.options; - const isColumns = Array.isArray(columns); - return { - ...this.__infoRecord(), - column: isColumns === true ? - (columns.length > this.state.record.length ? - columns[this.state.record.length].name : - null - ) : - this.state.record.length, - quoting: this.state.wasQuoting, - }; - } -} + }; +}; -const parse = function(data, options={}){ +const parse = function(data, opts={}){ if(typeof data === 'string'){ data = Buffer.from(data); } - const records = options && options.objname ? {} : []; - const parser = new Parser(options); - parser.push = function(record){ - if(record === null){ - return; - } - if(options.objname === undefined) + const records = opts && opts.objname ? {} : []; + const parser = transform$1(opts); + const push = (record) => { + if(parser.options.objname === undefined) records.push(record); else { records[record[0]] = record[1]; } }; - const err1 = parser.__parse(data, false); + const close = () => {}; + const err1 = parser.parse(data, false, push, close); if(err1 !== undefined) throw err1; - const err2 = parser.__parse(undefined, true); + const err2 = parser.parse(undefined, true, push, close); if(err2 !== undefined) throw err2; return records; }; -const bom_utf8 = Buffer.from([239, 187, 191]); - -class CsvError extends Error { - constructor(code, message, ...contexts) { - if(Array.isArray(message)) message = message.join(' '); - super(message); - if(Error.captureStackTrace !== undefined){ - Error.captureStackTrace(this, CsvError); - } - this.code = code; - for(const context of contexts){ - for(const key in context){ - const value = context[key]; - this[key] = isBuffer$1(value) ? value.toString() : value == null ? value : JSON.parse(JSON.stringify(value)); - } - } - } -} - -const isObject = function(obj){ - return typeof obj === 'object' && obj !== null && ! Array.isArray(obj); -}; - -const underscore = function(str){ - return str.replace(/([A-Z])/g, function(_, match){ - return '_' + match.toLowerCase(); - }); -}; - // Lodash implementation of `get` const charCodeOfDot = '.'.charCodeAt(0); @@ -6650,455 +6642,473 @@ const get = function(object, path){ return (index && index === length) ? object : undefined; }; -class Stringifier extends Transform { - constructor(opts = {}){ - super({...{writableObjectMode: true}, ...opts}); - const options = {}; - let err; - // Merge with user options - for(const opt in opts){ - options[underscore(opt)] = opts[opt]; - } - if((err = this.normalize(options)) !== undefined) throw err; - switch(options.record_delimiter){ - case 'auto': - options.record_delimiter = null; - break; - case 'unix': - options.record_delimiter = "\n"; - break; - case 'mac': - options.record_delimiter = "\r"; - break; - case 'windows': - options.record_delimiter = "\r\n"; - break; - case 'ascii': - options.record_delimiter = "\u001e"; - break; - case 'unicode': - options.record_delimiter = "\u2028"; - break; - } - // Expose options - this.options = options; - // Internal state - this.state = { - stop: false - }; - // Information - this.info = { - records: 0 - }; +const is_object = function(obj){ + return typeof obj === 'object' && obj !== null && ! Array.isArray(obj); +}; + +const normalize_columns = function(columns){ + if(columns === undefined || columns === null){ + return [undefined, undefined]; } - normalize(options){ - // Normalize option `bom` - if(options.bom === undefined || options.bom === null || options.bom === false){ - options.bom = false; - }else if(options.bom !== true){ - return new CsvError('CSV_OPTION_BOOLEAN_INVALID_TYPE', [ - 'option `bom` is optional and must be a boolean value,', - `got ${JSON.stringify(options.bom)}` - ]); - } - // Normalize option `delimiter` - if(options.delimiter === undefined || options.delimiter === null){ - options.delimiter = ','; - }else if(isBuffer$1(options.delimiter)){ - options.delimiter = options.delimiter.toString(); - }else if(typeof options.delimiter !== 'string'){ - return new CsvError('CSV_OPTION_DELIMITER_INVALID_TYPE', [ - 'option `delimiter` must be a buffer or a string,', - `got ${JSON.stringify(options.delimiter)}` - ]); - } - // Normalize option `quote` - if(options.quote === undefined || options.quote === null){ - options.quote = '"'; - }else if(options.quote === true){ - options.quote = '"'; - }else if(options.quote === false){ - options.quote = ''; - }else if (isBuffer$1(options.quote)){ - options.quote = options.quote.toString(); - }else if(typeof options.quote !== 'string'){ - return new CsvError('CSV_OPTION_QUOTE_INVALID_TYPE', [ - 'option `quote` must be a boolean, a buffer or a string,', - `got ${JSON.stringify(options.quote)}` - ]); + if(typeof columns !== 'object'){ + return [Error('Invalid option "columns": expect an array or an object')]; + } + if(!Array.isArray(columns)){ + const newcolumns = []; + for(const k in columns){ + newcolumns.push({ + key: k, + header: columns[k] + }); } - // Normalize option `quoted` - if(options.quoted === undefined || options.quoted === null){ - options.quoted = false; - } - // Normalize option `quoted_empty` - if(options.quoted_empty === undefined || options.quoted_empty === null){ - options.quoted_empty = undefined; - } - // Normalize option `quoted_match` - if(options.quoted_match === undefined || options.quoted_match === null || options.quoted_match === false){ - options.quoted_match = null; - }else if(!Array.isArray(options.quoted_match)){ - options.quoted_match = [options.quoted_match]; - } - if(options.quoted_match){ - for(const quoted_match of options.quoted_match){ - const isString = typeof quoted_match === 'string'; - const isRegExp = quoted_match instanceof RegExp; - if(!isString && !isRegExp){ - return Error(`Invalid Option: quoted_match must be a string or a regex, got ${JSON.stringify(quoted_match)}`); + columns = newcolumns; + }else { + const newcolumns = []; + for(const column of columns){ + if(typeof column === 'string'){ + newcolumns.push({ + key: column, + header: column + }); + }else if(typeof column === 'object' && column !== null && !Array.isArray(column)){ + if(!column.key){ + return [Error('Invalid column definition: property "key" is required')]; } + if(column.header === undefined){ + column.header = column.key; + } + newcolumns.push(column); + }else { + return [Error('Invalid column definition: expect a string or an object')]; } } - // Normalize option `quoted_string` - if(options.quoted_string === undefined || options.quoted_string === null){ - options.quoted_string = false; - } - // Normalize option `eof` - if(options.eof === undefined || options.eof === null){ - options.eof = true; - } - // Normalize option `escape` - if(options.escape === undefined || options.escape === null){ - options.escape = '"'; - }else if(isBuffer$1(options.escape)){ - options.escape = options.escape.toString(); - }else if(typeof options.escape !== 'string'){ - return Error(`Invalid Option: escape must be a buffer or a string, got ${JSON.stringify(options.escape)}`); - } - if (options.escape.length > 1){ - return Error(`Invalid Option: escape must be one character, got ${options.escape.length} characters`); - } - // Normalize option `header` - if(options.header === undefined || options.header === null){ - options.header = false; - } - // Normalize option `columns` - const [err, columns] = this.normalize_columns(options.columns); - if(err) return err; - options.columns = columns; - // Normalize option `quoted` - if(options.quoted === undefined || options.quoted === null){ - options.quoted = false; - } - // Normalize option `cast` - if(options.cast === undefined || options.cast === null){ - options.cast = {}; - } - // Normalize option cast.bigint - if(options.cast.bigint === undefined || options.cast.bigint === null){ - // Cast boolean to string by default - options.cast.bigint = value => '' + value; - } - // Normalize option cast.boolean - if(options.cast.boolean === undefined || options.cast.boolean === null){ - // Cast boolean to string by default - options.cast.boolean = value => value ? '1' : ''; - } - // Normalize option cast.date - if(options.cast.date === undefined || options.cast.date === null){ - // Cast date to timestamp string by default - options.cast.date = value => '' + value.getTime(); - } - // Normalize option cast.number - if(options.cast.number === undefined || options.cast.number === null){ - // Cast number to string using native casting by default - options.cast.number = value => '' + value; - } - // Normalize option cast.object - if(options.cast.object === undefined || options.cast.object === null){ - // Stringify object as JSON by default - options.cast.object = value => JSON.stringify(value); - } - // Normalize option cast.string - if(options.cast.string === undefined || options.cast.string === null){ - // Leave string untouched - options.cast.string = function(value){return value;}; - } - // Normalize option `record_delimiter` - if(options.record_delimiter === undefined || options.record_delimiter === null){ - options.record_delimiter = '\n'; - }else if(isBuffer$1(options.record_delimiter)){ - options.record_delimiter = options.record_delimiter.toString(); - }else if(typeof options.record_delimiter !== 'string'){ - return Error(`Invalid Option: record_delimiter must be a buffer or a string, got ${JSON.stringify(options.record_delimiter)}`); - } - } - _transform(chunk, encoding, callback){ - if(this.state.stop === true){ - return; - } - const err = this.__transform(chunk); - if(err !== undefined){ - this.state.stop = true; - } - callback(err); + columns = newcolumns; } - _flush(callback){ - if(this.state.stop === true){ - // Note, Node.js 12 call flush even after an error, we must prevent - // `callback` from being called in flush without any error. - return; + return [undefined, columns]; +}; + +class CsvError extends Error { + constructor(code, message, ...contexts) { + if(Array.isArray(message)) message = message.join(' '); + super(message); + if(Error.captureStackTrace !== undefined){ + Error.captureStackTrace(this, CsvError); } - if(this.info.records === 0){ - this.bom(); - const err = this.headers(); - if(err) callback(err); + this.code = code; + for(const context of contexts){ + for(const key in context){ + const value = context[key]; + this[key] = isBuffer$1(value) ? value.toString() : value == null ? value : JSON.parse(JSON.stringify(value)); + } } - callback(); } - __transform(chunk){ - // Chunk validation - if(!Array.isArray(chunk) && typeof chunk !== 'object'){ - return Error(`Invalid Record: expect an array or an object, got ${JSON.stringify(chunk)}`); - } - // Detect columns from the first record - if(this.info.records === 0){ - if(Array.isArray(chunk)){ - if(this.options.header === true && this.options.columns === undefined){ - return Error('Undiscoverable Columns: header option requires column option or object records'); - } - }else if(this.options.columns === undefined){ - const [err, columns] = this.normalize_columns(Object.keys(chunk)); - if(err) return; - this.options.columns = columns; +} + +const underscore = function(str){ + return str.replace(/([A-Z])/g, function(_, match){ + return '_' + match.toLowerCase(); + }); +}; + +const normalize_options = function(opts) { + const options = {}; + // Merge with user options + for(const opt in opts){ + options[underscore(opt)] = opts[opt]; + } + // Normalize option `bom` + if(options.bom === undefined || options.bom === null || options.bom === false){ + options.bom = false; + }else if(options.bom !== true){ + return [new CsvError('CSV_OPTION_BOOLEAN_INVALID_TYPE', [ + 'option `bom` is optional and must be a boolean value,', + `got ${JSON.stringify(options.bom)}` + ])]; + } + // Normalize option `delimiter` + if(options.delimiter === undefined || options.delimiter === null){ + options.delimiter = ','; + }else if(isBuffer$1(options.delimiter)){ + options.delimiter = options.delimiter.toString(); + }else if(typeof options.delimiter !== 'string'){ + return [new CsvError('CSV_OPTION_DELIMITER_INVALID_TYPE', [ + 'option `delimiter` must be a buffer or a string,', + `got ${JSON.stringify(options.delimiter)}` + ])]; + } + // Normalize option `quote` + if(options.quote === undefined || options.quote === null){ + options.quote = '"'; + }else if(options.quote === true){ + options.quote = '"'; + }else if(options.quote === false){ + options.quote = ''; + }else if (isBuffer$1(options.quote)){ + options.quote = options.quote.toString(); + }else if(typeof options.quote !== 'string'){ + return [new CsvError('CSV_OPTION_QUOTE_INVALID_TYPE', [ + 'option `quote` must be a boolean, a buffer or a string,', + `got ${JSON.stringify(options.quote)}` + ])]; + } + // Normalize option `quoted` + if(options.quoted === undefined || options.quoted === null){ + options.quoted = false; + } + // Normalize option `quoted_empty` + if(options.quoted_empty === undefined || options.quoted_empty === null){ + options.quoted_empty = undefined; + } + // Normalize option `quoted_match` + if(options.quoted_match === undefined || options.quoted_match === null || options.quoted_match === false){ + options.quoted_match = null; + }else if(!Array.isArray(options.quoted_match)){ + options.quoted_match = [options.quoted_match]; + } + if(options.quoted_match){ + for(const quoted_match of options.quoted_match){ + const isString = typeof quoted_match === 'string'; + const isRegExp = quoted_match instanceof RegExp; + if(!isString && !isRegExp){ + return [Error(`Invalid Option: quoted_match must be a string or a regex, got ${JSON.stringify(quoted_match)}`)]; } } - // Emit the header - if(this.info.records === 0){ - this.bom(); - const err = this.headers(); - if(err) return err; - } - // Emit and stringify the record if an object or an array - try{ - this.emit('record', chunk, this.info.records); - }catch(err){ - return err; - } - // Convert the record into a string - let err, chunk_string; - if(this.options.eof){ - [err, chunk_string] = this.stringify(chunk); - if(err) return err; - if(chunk_string === undefined){ - return; - }else { - chunk_string = chunk_string + this.options.record_delimiter; + } + // Normalize option `quoted_string` + if(options.quoted_string === undefined || options.quoted_string === null){ + options.quoted_string = false; + } + // Normalize option `eof` + if(options.eof === undefined || options.eof === null){ + options.eof = true; + } + // Normalize option `escape` + if(options.escape === undefined || options.escape === null){ + options.escape = '"'; + }else if(isBuffer$1(options.escape)){ + options.escape = options.escape.toString(); + }else if(typeof options.escape !== 'string'){ + return [Error(`Invalid Option: escape must be a buffer or a string, got ${JSON.stringify(options.escape)}`)]; + } + if (options.escape.length > 1){ + return [Error(`Invalid Option: escape must be one character, got ${options.escape.length} characters`)]; + } + // Normalize option `header` + if(options.header === undefined || options.header === null){ + options.header = false; + } + // Normalize option `columns` + const [errColumns, columns] = normalize_columns(options.columns); + if(errColumns !== undefined) return [errColumns]; + options.columns = columns; + // Normalize option `quoted` + if(options.quoted === undefined || options.quoted === null){ + options.quoted = false; + } + // Normalize option `cast` + if(options.cast === undefined || options.cast === null){ + options.cast = {}; + } + // Normalize option cast.bigint + if(options.cast.bigint === undefined || options.cast.bigint === null){ + // Cast boolean to string by default + options.cast.bigint = value => '' + value; + } + // Normalize option cast.boolean + if(options.cast.boolean === undefined || options.cast.boolean === null){ + // Cast boolean to string by default + options.cast.boolean = value => value ? '1' : ''; + } + // Normalize option cast.date + if(options.cast.date === undefined || options.cast.date === null){ + // Cast date to timestamp string by default + options.cast.date = value => '' + value.getTime(); + } + // Normalize option cast.number + if(options.cast.number === undefined || options.cast.number === null){ + // Cast number to string using native casting by default + options.cast.number = value => '' + value; + } + // Normalize option cast.object + if(options.cast.object === undefined || options.cast.object === null){ + // Stringify object as JSON by default + options.cast.object = value => JSON.stringify(value); + } + // Normalize option cast.string + if(options.cast.string === undefined || options.cast.string === null){ + // Leave string untouched + options.cast.string = function(value){return value;}; + } + // Normalize option `on_record` + if(options.on_record !== undefined && typeof options.on_record !== 'function'){ + return [Error(`Invalid Option: "on_record" must be a function.`)]; + } + // Normalize option `record_delimiter` + if(options.record_delimiter === undefined || options.record_delimiter === null){ + options.record_delimiter = '\n'; + }else if(isBuffer$1(options.record_delimiter)){ + options.record_delimiter = options.record_delimiter.toString(); + }else if(typeof options.record_delimiter !== 'string'){ + return [Error(`Invalid Option: record_delimiter must be a buffer or a string, got ${JSON.stringify(options.record_delimiter)}`)]; + } + switch(options.record_delimiter){ + case 'auto': + options.record_delimiter = null; + break; + case 'unix': + options.record_delimiter = "\n"; + break; + case 'mac': + options.record_delimiter = "\r"; + break; + case 'windows': + options.record_delimiter = "\r\n"; + break; + case 'ascii': + options.record_delimiter = "\u001e"; + break; + case 'unicode': + options.record_delimiter = "\u2028"; + break; + } + return [undefined, options]; +}; + +const bom_utf8 = Buffer.from([239, 187, 191]); + +const stringifier = function(options, state, info){ + return { + options: options, + state: state, + info: info, + __transform: function(chunk, push){ + // Chunk validation + if(!Array.isArray(chunk) && typeof chunk !== 'object'){ + return Error(`Invalid Record: expect an array or an object, got ${JSON.stringify(chunk)}`); } - }else { - [err, chunk_string] = this.stringify(chunk); - if(err) return err; - if(chunk_string === undefined){ - return; - }else { - if(this.options.header || this.info.records){ - chunk_string = this.options.record_delimiter + chunk_string; + // Detect columns from the first record + if(this.info.records === 0){ + if(Array.isArray(chunk)){ + if(this.options.header === true && this.options.columns === undefined){ + return Error('Undiscoverable Columns: header option requires column option or object records'); + } + }else if(this.options.columns === undefined){ + const [err, columns] = normalize_columns(Object.keys(chunk)); + if(err) return; + this.options.columns = columns; } } - } - // Emit the csv - this.info.records++; - this.push(chunk_string); - } - stringify(chunk, chunkIsHeader=false){ - if(typeof chunk !== 'object'){ - return [undefined, chunk]; - } - const {columns} = this.options; - const record = []; - // Record is an array - if(Array.isArray(chunk)){ - // We are getting an array but the user has specified output columns. In - // this case, we respect the columns indexes - if(columns){ - chunk.splice(columns.length); + // Emit the header + if(this.info.records === 0){ + this.bom(push); + const err = this.headers(push); + if(err) return err; } - // Cast record elements - for(let i=0; i= 0; - const containsQuote = (quote !== '') && value.indexOf(quote) >= 0; - const containsEscape = value.indexOf(escape) >= 0 && (escape !== quote); - const containsRecordDelimiter = value.indexOf(record_delimiter) >= 0; - const quotedString = quoted_string && typeof field === 'string'; - let quotedMatch = quoted_match && quoted_match.filter(quoted_match => { - if(typeof quoted_match === 'string'){ - return value.indexOf(quoted_match) !== -1; - }else { - return quoted_match.test(value); + } + let csvrecord = ''; + for(let i=0; i 0; - const shouldQuote = containsQuote === true || containsdelimiter || containsRecordDelimiter || quoted || quotedString || quotedMatch; - if(shouldQuote === true && containsEscape === true){ - const regexp = escape === '\\' - ? new RegExp(escape + escape, 'g') - : new RegExp(escape, 'g'); - value = value.replace(regexp, escape + escape); + options = {...this.options, ...options}; + [err, options] = normalize_options(options); + if(err !== undefined){ + return [err]; + } + }else if(value === undefined || value === null){ + options = this.options; + }else { + return [Error(`Invalid Casting Value: returned value must return a string, an object, null or undefined, got ${JSON.stringify(value)}`)]; } - if(containsQuote === true){ - const regexp = new RegExp(quote,'g'); - value = value.replace(regexp, escape + quote); + const {delimiter, escape, quote, quoted, quoted_empty, quoted_string, quoted_match, record_delimiter} = options; + if(value){ + if(typeof value !== 'string'){ + return [Error(`Formatter must return a string, null or undefined, got ${JSON.stringify(value)}`)]; + } + const containsdelimiter = delimiter.length && value.indexOf(delimiter) >= 0; + const containsQuote = (quote !== '') && value.indexOf(quote) >= 0; + const containsEscape = value.indexOf(escape) >= 0 && (escape !== quote); + const containsRecordDelimiter = value.indexOf(record_delimiter) >= 0; + const quotedString = quoted_string && typeof field === 'string'; + let quotedMatch = quoted_match && quoted_match.filter(quoted_match => { + if(typeof quoted_match === 'string'){ + return value.indexOf(quoted_match) !== -1; + }else { + return quoted_match.test(value); + } + }); + quotedMatch = quotedMatch && quotedMatch.length > 0; + const shouldQuote = containsQuote === true || containsdelimiter || containsRecordDelimiter || quoted || quotedString || quotedMatch; + if(shouldQuote === true && containsEscape === true){ + const regexp = escape === '\\' + ? new RegExp(escape + escape, 'g') + : new RegExp(escape, 'g'); + value = value.replace(regexp, escape + escape); + } + if(containsQuote === true){ + const regexp = new RegExp(quote,'g'); + value = value.replace(regexp, escape + quote); + } + if(shouldQuote === true){ + value = quote + value + quote; + } + csvrecord += value; + }else if(quoted_empty === true || (field === '' && quoted_string === true && quoted_empty !== false)){ + csvrecord += quote + quote; } - if(shouldQuote === true){ - value = quote + value + quote; + if(i !== record.length - 1){ + csvrecord += delimiter; } - csvrecord += value; - }else if(quoted_empty === true || (field === '' && quoted_string === true && quoted_empty !== false)){ - csvrecord += quote + quote; } - if(i !== record.length - 1){ - csvrecord += delimiter; + return [undefined, csvrecord]; + }, + bom: function(push){ + if(this.options.bom !== true){ + return; } - } - return [undefined, csvrecord]; - } - bom(){ - if(this.options.bom !== true){ - return; - } - this.push(bom_utf8); - } - headers(){ - if(this.options.header === false){ - return; - } - if(this.options.columns === undefined){ - return; - } - let err; - let headers = this.options.columns.map(column => column.header); - if(this.options.eof){ - [err, headers] = this.stringify(headers, true); - headers += this.options.record_delimiter; - }else { - [err, headers] = this.stringify(headers); - } - if(err) return err; - this.push(headers); - } - __cast(value, context){ - const type = typeof value; - try{ - if(type === 'string'){ // Fine for 99% of the cases - return [undefined, this.options.cast.string(value, context)]; - }else if(type === 'bigint'){ - return [undefined, this.options.cast.bigint(value, context)]; - }else if(type === 'number'){ - return [undefined, this.options.cast.number(value, context)]; - }else if(type === 'boolean'){ - return [undefined, this.options.cast.boolean(value, context)]; - }else if(value instanceof Date){ - return [undefined, this.options.cast.date(value, context)]; - }else if(type === 'object' && value !== null){ - return [undefined, this.options.cast.object(value, context)]; - }else { - return [undefined, value, value]; + push(bom_utf8); + }, + headers: function(push){ + if(this.options.header === false){ + return; } - }catch(err){ - return [err]; - } - } - normalize_columns(columns){ - if(columns === undefined || columns === null){ - return []; - } - if(typeof columns !== 'object'){ - return [Error('Invalid option "columns": expect an array or an object')]; - } - if(!Array.isArray(columns)){ - const newcolumns = []; - for(const k in columns){ - newcolumns.push({ - key: k, - header: columns[k] - }); + if(this.options.columns === undefined){ + return; } - columns = newcolumns; - }else { - const newcolumns = []; - for(const column of columns){ - if(typeof column === 'string'){ - newcolumns.push({ - key: column, - header: column - }); - }else if(typeof column === 'object' && column !== undefined && !Array.isArray(column)){ - if(!column.key){ - return [Error('Invalid column definition: property "key" is required')]; - } - if(column.header === undefined){ - column.header = column.key; - } - newcolumns.push(column); + let err; + let headers = this.options.columns.map(column => column.header); + if(this.options.eof){ + [err, headers] = this.stringify(headers, true); + headers += this.options.record_delimiter; + }else { + [err, headers] = this.stringify(headers); + } + if(err) return err; + push(headers); + }, + __cast: function(value, context){ + const type = typeof value; + try{ + if(type === 'string'){ // Fine for 99% of the cases + return [undefined, this.options.cast.string(value, context)]; + }else if(type === 'bigint'){ + return [undefined, this.options.cast.bigint(value, context)]; + }else if(type === 'number'){ + return [undefined, this.options.cast.number(value, context)]; + }else if(type === 'boolean'){ + return [undefined, this.options.cast.boolean(value, context)]; + }else if(value instanceof Date){ + return [undefined, this.options.cast.date(value, context)]; + }else if(type === 'object' && value !== null){ + return [undefined, this.options.cast.object(value, context)]; }else { - return [Error('Invalid column definition: expect a string or an object')]; + return [undefined, value, value]; } + }catch(err){ + return [err]; } - columns = newcolumns; } - return [undefined, columns]; - } -} + }; +}; -const stringify = function(records, options={}){ +const stringify = function(records, opts={}){ const data = []; - const stringifier = new Stringifier(options); - stringifier.push = function(record){ - if(record === null){ - return; - } - data.push(record.toString()); + const [err, options] = normalize_options(opts); + if(err !== undefined) throw err; + const state = { + stop: false + }; + // Information + const info = { + records: 0 }; + const api = stringifier(options, state, info); + // stringifier.push = function(record){ + // if(record === null){ + // return; + // } + // data.push(record.toString()); + // }; for(const record of records){ - const err = stringifier.__transform(record, null); + const err = api.__transform(record, function(record){ + data.push(record); + }); if(err !== undefined) throw err; } return data.join(''); diff --git a/packages/csv/dist/iife/index.js b/packages/csv/dist/iife/index.js index 7cb7e456a..b21cd9b47 100644 --- a/packages/csv/dist/iife/index.js +++ b/packages/csv/dist/iife/index.js @@ -115,7 +115,7 @@ var csv = (function (exports) { return parts.join('') } - function read (buffer, offset, isLE, mLen, nBytes) { + function read$1 (buffer, offset, isLE, mLen, nBytes) { var e, m; var eLen = nBytes * 8 - mLen - 1; var eMax = (1 << eLen) - 1; @@ -1396,22 +1396,22 @@ var csv = (function (exports) { Buffer.prototype.readFloatLE = function readFloatLE (offset, noAssert) { if (!noAssert) checkOffset(offset, 4, this.length); - return read(this, offset, true, 23, 4) + return read$1(this, offset, true, 23, 4) }; Buffer.prototype.readFloatBE = function readFloatBE (offset, noAssert) { if (!noAssert) checkOffset(offset, 4, this.length); - return read(this, offset, false, 23, 4) + return read$1(this, offset, false, 23, 4) }; Buffer.prototype.readDoubleLE = function readDoubleLE (offset, noAssert) { if (!noAssert) checkOffset(offset, 8, this.length); - return read(this, offset, true, 52, 8) + return read$1(this, offset, true, 52, 8) }; Buffer.prototype.readDoubleBE = function readDoubleBE (offset, noAssert) { if (!noAssert) checkOffset(offset, 8, this.length); - return read(this, offset, false, 52, 8) + return read$1(this, offset, false, 52, 8) }; function checkInt (buf, value, offset, ext, max, min) { @@ -2630,7 +2630,7 @@ var csv = (function (exports) { } }); for (var x = args[i]; i < len; x = args[++i]) { - if (isNull(x) || !isObject$2(x)) { + if (isNull(x) || !isObject(x)) { str += ' ' + x; } else { str += ' ' + inspect(x); @@ -3046,19 +3046,19 @@ var csv = (function (exports) { } function isRegExp(re) { - return isObject$2(re) && objectToString(re) === '[object RegExp]'; + return isObject(re) && objectToString(re) === '[object RegExp]'; } - function isObject$2(arg) { + function isObject(arg) { return typeof arg === 'object' && arg !== null; } function isDate(d) { - return isObject$2(d) && objectToString(d) === '[object Date]'; + return isObject(d) && objectToString(d) === '[object Date]'; } function isError(e) { - return isObject$2(e) && + return isObject(e) && (objectToString(e) === '[object Error]' || e instanceof Error); } @@ -3109,7 +3109,7 @@ var csv = (function (exports) { function _extend(origin, add) { // Don't do anything if add isn't an object - if (!add || !isObject$2(add)) return origin; + if (!add || !isObject(add)) return origin; var keys = Object.keys(add); var i = keys.length; @@ -3131,7 +3131,7 @@ var csv = (function (exports) { isFunction: isFunction, isError: isError, isDate: isDate, - isObject: isObject$2, + isObject: isObject, isRegExp: isRegExp, isUndefined: isUndefined, isSymbol: isSymbol$1, @@ -5039,20 +5039,64 @@ var csv = (function (exports) { return dest; }; - const Generator = function(options = {}){ + const init_state$1 = (options) => { + // State + return { + start_time: options.duration ? Date.now() : null, + fixed_size_buffer: '', + count_written: 0, + count_created: 0, + }; + }; + + // Generate a random number between 0 and 1 with 2 decimals. The function is idempotent if it detect the "seed" option. + const random = function(options={}){ + if(options.seed){ + return options.seed = options.seed * Math.PI * 100 % 100 / 100; + }else { + return Math.random(); + } + }; + + const types = { + // Generate an ASCII value. + ascii: function({options}){ + const column = []; + const nb_chars = Math.ceil(random(options) * options.maxWordLength); + for(let i=0; i { // Convert Stream Readable options if underscored - if(options.high_water_mark){ - options.highWaterMark = options.high_water_mark; + if(opts.high_water_mark){ + opts.highWaterMark = opts.high_water_mark; } - if(options.object_mode){ - options.objectMode = options.object_mode; + if(opts.object_mode){ + opts.objectMode = opts.object_mode; } - // Call parent constructor - Stream.Readable.call(this, options); // Clone and camelize options - this.options = {}; - for(const k in options){ - this.options[Generator.camelize(k)] = options[k]; + const options = {}; + for(const k in opts){ + options[camelize(k)] = opts[k]; } // Normalize options const dft = { @@ -5070,110 +5114,100 @@ var csv = (function (exports) { sleep: 0, }; for(const k in dft){ - if(this.options[k] === undefined){ - this.options[k] = dft[k]; + if(options[k] === undefined){ + options[k] = dft[k]; } } // Default values - if(this.options.eof === true){ - this.options.eof = this.options.rowDelimiter; + if(options.eof === true){ + options.eof = options.rowDelimiter; } - // State - this._ = { - start_time: this.options.duration ? Date.now() : null, - fixed_size_buffer: '', - count_written: 0, - count_created: 0, - }; - if(typeof this.options.columns === 'number'){ - this.options.columns = new Array(this.options.columns); + if(typeof options.columns === 'number'){ + options.columns = new Array(options.columns); } - const accepted_header_types = Object.keys(Generator).filter((t) => (!['super_', 'camelize'].includes(t))); - for(let i = 0; i < this.options.columns.length; i++){ - const v = this.options.columns[i] || 'ascii'; + const accepted_header_types = Object.keys(types).filter((t) => (!['super_', 'camelize'].includes(t))); + for(let i = 0; i < options.columns.length; i++){ + const v = options.columns[i] || 'ascii'; if(typeof v === 'string'){ if(!accepted_header_types.includes(v)){ throw Error(`Invalid column type: got "${v}", default values are ${JSON.stringify(accepted_header_types)}`); } - this.options.columns[i] = Generator[v]; + options.columns[i] = types[v]; } } - return this; + return options; }; - util.inherits(Generator, Stream.Readable); - // Generate a random number between 0 and 1 with 2 decimals. The function is idempotent if it detect the "seed" option. - Generator.prototype.random = function(){ - if(this.options.seed){ - return this.options.seed = this.options.seed * Math.PI * 100 % 100 / 100; - }else { - return Math.random(); - } - }; - // Stop the generation. - Generator.prototype.end = function(){ - this.push(null); - }; - // Put new data into the read queue. - Generator.prototype._read = function(size){ + const read = (options, state, size, push, close) => { // Already started const data = []; - let length = this._.fixed_size_buffer.length; + let length = state.fixed_size_buffer.length; if(length !== 0){ - data.push(this._.fixed_size_buffer); + data.push(state.fixed_size_buffer); } // eslint-disable-next-line while(true){ // Time for some rest: flush first and stop later - if((this._.count_created === this.options.length) || (this.options.end && Date.now() > this.options.end) || (this.options.duration && Date.now() > this._.start_time + this.options.duration)){ + if((state.count_created === options.length) || (options.end && Date.now() > options.end) || (options.duration && Date.now() > state.start_time + options.duration)){ // Flush if(data.length){ - if(this.options.objectMode){ + if(options.objectMode){ for(const record of data){ - this.__push(record); + push(record); } }else { - this.__push(data.join('') + (this.options.eof ? this.options.eof : '')); + push(data.join('') + (options.eof ? options.eof : '')); } - this._.end = true; + state.end = true; }else { - this.push(null); + close(); } return; } // Create the record let record = []; let recordLength; - this.options.columns.forEach((fn) => { - record.push(fn(this)); + options.columns.forEach((fn) => { + const result = fn({options: options, state: state}); + const type = typeof result; + if(result !== null && type !== 'string' && type !== 'number'){ + throw Error([ + 'INVALID_VALUE:', + 'values returned by column function must be', + 'a string, a number or null,', + `got ${JSON.stringify(result)}` + ].join(' ')); + } + record.push(result); }); // Obtain record length - if(this.options.objectMode){ + if(options.objectMode){ recordLength = 0; // recordLength is currently equal to the number of columns // This is wrong and shall equal to 1 record only - for(const column of record) + for(const column of record){ recordLength += column.length; + } }else { // Stringify the record - record = (this._.count_created === 0 ? '' : this.options.rowDelimiter)+record.join(this.options.delimiter); + record = (state.count_created === 0 ? '' : options.rowDelimiter)+record.join(options.delimiter); recordLength = record.length; } - this._.count_created++; + state.count_created++; if(length + recordLength > size){ - if(this.options.objectMode){ + if(options.objectMode){ data.push(record); for(const record of data){ - this.__push(record); + push(record); } }else { - if(this.options.fixedSize){ - this._.fixed_size_buffer = record.substr(size - length); + if(options.fixedSize){ + state.fixed_size_buffer = record.substr(size - length); data.push(record.substr(0, size - length)); }else { data.push(record); } - this.__push(data.join('')); + push(data.join('')); } return; } @@ -5181,42 +5215,41 @@ var csv = (function (exports) { data.push(record); } }; + + const Generator = function(options = {}){ + this.options = normalize_options$2(options); + // Call parent constructor + Stream.Readable.call(this, this.options); + this.state = init_state$1(this.options); + return this; + }; + util.inherits(Generator, Stream.Readable); + + // Stop the generation. + Generator.prototype.end = function(){ + this.push(null); + }; + // Put new data into the read queue. + Generator.prototype._read = function(size){ + const self = this; + read(this.options, this.state, size, function(chunk) { + self.__push(chunk); + }, function(){ + self.push(null); + }); + }; // Put new data into the read queue. Generator.prototype.__push = function(record){ + // console.log('push', record) const push = () => { - this._.count_written++; + this.state.count_written++; this.push(record); - if(this._.end === true){ + if(this.state.end === true){ return this.push(null); } }; this.options.sleep > 0 ? setTimeout(push, this.options.sleep) : push(); }; - // Generate an ASCII value. - Generator.ascii = function(gen){ - // Column - const column = []; - const nb_chars = Math.ceil(gen.random() * gen.options.maxWordLength); - for(let i=0; i delimiter.length), + // Skip if the remaining buffer can be escape sequence + options.quote !== null ? options.quote.length : 0, + ), + previousBuf: undefined, + quoting: false, + stop: false, + rawBuffer: new ResizeableBuffer(100), + record: [], + recordHasError: false, + record_length: 0, + recordDelimiterMaxLength: options.record_delimiter.length === 0 ? 2 : Math.max(...options.record_delimiter.map((v) => v.length)), + trimChars: [Buffer.from(' ', options.encoding)[0], Buffer.from('\t', options.encoding)[0]], + wasQuoting: false, + wasRowDelimiter: false + }; }; - class CsvError$1 extends Error { - constructor(code, message, options, ...contexts) { - if(Array.isArray(message)) message = message.join(' '); - super(message); - if(Error.captureStackTrace !== undefined){ - Error.captureStackTrace(this, CsvError$1); - } - this.code = code; - for(const context of contexts){ - for(const key in context){ - const value = context[key]; - this[key] = isBuffer$1(value) ? value.toString(options.encoding) : value == null ? value : JSON.parse(JSON.stringify(value)); - } - } - } - } - const underscore$1 = function(str){ return str.replace(/([A-Z])/g, function(_, match){ return '_' + match.toLowerCase(); }); }; - const isObject$1 = function(obj){ - return (typeof obj === 'object' && obj !== null && !Array.isArray(obj)); - }; - - const isRecordEmpty = function(record){ - return record.every((field) => field == null || field.toString && field.toString().trim() === ''); - }; - - const normalizeColumnsArray = function(columns){ - const normalizedColumns = []; - for(let i = 0, l = columns.length; i < l; i++){ - const column = columns[i]; - if(column === undefined || column === null || column === false){ - normalizedColumns[i] = { disabled: true }; - }else if(typeof column === 'string'){ - normalizedColumns[i] = { name: column }; - }else if(isObject$1(column)){ - if(typeof column.name !== 'string'){ - throw new CsvError$1('CSV_OPTION_COLUMNS_MISSING_NAME', [ - 'Option columns missing name:', - `property "name" is required at position ${i}`, - 'when column is an object literal' - ]); - } - normalizedColumns[i] = column; - }else { - throw new CsvError$1('CSV_INVALID_COLUMN_DEFINITION', [ - 'Invalid column definition:', - 'expect a string or a literal object,', - `got ${JSON.stringify(column)} at position ${i}` - ]); - } - } - return normalizedColumns; - }; - - class Parser extends Transform { - constructor(opts = {}){ - super({...{readableObjectMode: true}, ...opts, encoding: null}); - this.__originalOptions = opts; - this.__normalizeOptions(opts); - } - __normalizeOptions(opts){ - const options = {}; - // Merge with user options - for(const opt in opts){ - options[underscore$1(opt)] = opts[opt]; - } - // Normalize option `encoding` - // Note: defined first because other options depends on it - // to convert chars/strings into buffers. - if(options.encoding === undefined || options.encoding === true){ - options.encoding = 'utf8'; - }else if(options.encoding === null || options.encoding === false){ - options.encoding = null; - }else if(typeof options.encoding !== 'string' && options.encoding !== null){ - throw new CsvError$1('CSV_INVALID_OPTION_ENCODING', [ - 'Invalid option encoding:', - 'encoding must be a string or null to return a buffer,', - `got ${JSON.stringify(options.encoding)}` - ], options); - } - // Normalize option `bom` - if(options.bom === undefined || options.bom === null || options.bom === false){ - options.bom = false; - }else if(options.bom !== true){ - throw new CsvError$1('CSV_INVALID_OPTION_BOM', [ - 'Invalid option bom:', 'bom must be true,', - `got ${JSON.stringify(options.bom)}` - ], options); - } - // Normalize option `cast` - let fnCastField = null; - if(options.cast === undefined || options.cast === null || options.cast === false || options.cast === ''){ - options.cast = undefined; - }else if(typeof options.cast === 'function'){ - fnCastField = options.cast; - options.cast = true; - }else if(options.cast !== true){ - throw new CsvError$1('CSV_INVALID_OPTION_CAST', [ - 'Invalid option cast:', 'cast must be true or a function,', - `got ${JSON.stringify(options.cast)}` - ], options); - } - // Normalize option `cast_date` - if(options.cast_date === undefined || options.cast_date === null || options.cast_date === false || options.cast_date === ''){ - options.cast_date = false; - }else if(options.cast_date === true){ - options.cast_date = function(value){ - const date = Date.parse(value); - return !isNaN(date) ? new Date(date) : value; - }; - }else { - throw new CsvError$1('CSV_INVALID_OPTION_CAST_DATE', [ - 'Invalid option cast_date:', 'cast_date must be true or a function,', - `got ${JSON.stringify(options.cast_date)}` - ], options); - } - // Normalize option `columns` - let fnFirstLineToHeaders = null; - if(options.columns === true){ - // Fields in the first line are converted as-is to columns - fnFirstLineToHeaders = undefined; - }else if(typeof options.columns === 'function'){ - fnFirstLineToHeaders = options.columns; - options.columns = true; - }else if(Array.isArray(options.columns)){ - options.columns = normalizeColumnsArray(options.columns); - }else if(options.columns === undefined || options.columns === null || options.columns === false){ - options.columns = false; - }else { - throw new CsvError$1('CSV_INVALID_OPTION_COLUMNS', [ - 'Invalid option columns:', - 'expect an array, a function or true,', - `got ${JSON.stringify(options.columns)}` + const normalize_options$1 = function(opts){ + const options = {}; + // Merge with user options + for(const opt in opts){ + options[underscore$1(opt)] = opts[opt]; + } + // Normalize option `encoding` + // Note: defined first because other options depends on it + // to convert chars/strings into buffers. + if(options.encoding === undefined || options.encoding === true){ + options.encoding = 'utf8'; + }else if(options.encoding === null || options.encoding === false){ + options.encoding = null; + }else if(typeof options.encoding !== 'string' && options.encoding !== null){ + throw new CsvError$1('CSV_INVALID_OPTION_ENCODING', [ + 'Invalid option encoding:', + 'encoding must be a string or null to return a buffer,', + `got ${JSON.stringify(options.encoding)}` + ], options); + } + // Normalize option `bom` + if(options.bom === undefined || options.bom === null || options.bom === false){ + options.bom = false; + }else if(options.bom !== true){ + throw new CsvError$1('CSV_INVALID_OPTION_BOM', [ + 'Invalid option bom:', 'bom must be true,', + `got ${JSON.stringify(options.bom)}` + ], options); + } + // Normalize option `cast` + options.cast_function = null; + if(options.cast === undefined || options.cast === null || options.cast === false || options.cast === ''){ + options.cast = undefined; + }else if(typeof options.cast === 'function'){ + options.cast_function = options.cast; + options.cast = true; + }else if(options.cast !== true){ + throw new CsvError$1('CSV_INVALID_OPTION_CAST', [ + 'Invalid option cast:', 'cast must be true or a function,', + `got ${JSON.stringify(options.cast)}` + ], options); + } + // Normalize option `cast_date` + if(options.cast_date === undefined || options.cast_date === null || options.cast_date === false || options.cast_date === ''){ + options.cast_date = false; + }else if(options.cast_date === true){ + options.cast_date = function(value){ + const date = Date.parse(value); + return !isNaN(date) ? new Date(date) : value; + }; + }else { + throw new CsvError$1('CSV_INVALID_OPTION_CAST_DATE', [ + 'Invalid option cast_date:', 'cast_date must be true or a function,', + `got ${JSON.stringify(options.cast_date)}` + ], options); + } + // Normalize option `columns` + options.cast_first_line_to_header = null; + if(options.columns === true){ + // Fields in the first line are converted as-is to columns + options.cast_first_line_to_header = undefined; + }else if(typeof options.columns === 'function'){ + options.cast_first_line_to_header = options.columns; + options.columns = true; + }else if(Array.isArray(options.columns)){ + options.columns = normalize_columns_array(options.columns); + }else if(options.columns === undefined || options.columns === null || options.columns === false){ + options.columns = false; + }else { + throw new CsvError$1('CSV_INVALID_OPTION_COLUMNS', [ + 'Invalid option columns:', + 'expect an array, a function or true,', + `got ${JSON.stringify(options.columns)}` + ], options); + } + // Normalize option `group_columns_by_name` + if(options.group_columns_by_name === undefined || options.group_columns_by_name === null || options.group_columns_by_name === false){ + options.group_columns_by_name = false; + }else if(options.group_columns_by_name !== true){ + throw new CsvError$1('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ + 'Invalid option group_columns_by_name:', + 'expect an boolean,', + `got ${JSON.stringify(options.group_columns_by_name)}` + ], options); + }else if(options.columns === false){ + throw new CsvError$1('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ + 'Invalid option group_columns_by_name:', + 'the `columns` mode must be activated.' + ], options); + } + // Normalize option `comment` + if(options.comment === undefined || options.comment === null || options.comment === false || options.comment === ''){ + options.comment = null; + }else { + if(typeof options.comment === 'string'){ + options.comment = Buffer.from(options.comment, options.encoding); + } + if(!isBuffer$1(options.comment)){ + throw new CsvError$1('CSV_INVALID_OPTION_COMMENT', [ + 'Invalid option comment:', + 'comment must be a buffer or a string,', + `got ${JSON.stringify(options.comment)}` ], options); } - // Normalize option `group_columns_by_name` - if(options.group_columns_by_name === undefined || options.group_columns_by_name === null || options.group_columns_by_name === false){ - options.group_columns_by_name = false; - }else if(options.group_columns_by_name !== true){ - throw new CsvError$1('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ - 'Invalid option group_columns_by_name:', - 'expect an boolean,', - `got ${JSON.stringify(options.group_columns_by_name)}` - ], options); - }else if(options.columns === false){ - throw new CsvError$1('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ - 'Invalid option group_columns_by_name:', - 'the `columns` mode must be activated.' - ], options); + } + // Normalize option `delimiter` + const delimiter_json = JSON.stringify(options.delimiter); + if(!Array.isArray(options.delimiter)) options.delimiter = [options.delimiter]; + if(options.delimiter.length === 0){ + throw new CsvError$1('CSV_INVALID_OPTION_DELIMITER', [ + 'Invalid option delimiter:', + 'delimiter must be a non empty string or buffer or array of string|buffer,', + `got ${delimiter_json}` + ], options); + } + options.delimiter = options.delimiter.map(function(delimiter){ + if(delimiter === undefined || delimiter === null || delimiter === false){ + return Buffer.from(',', options.encoding); } - // Normalize option `comment` - if(options.comment === undefined || options.comment === null || options.comment === false || options.comment === ''){ - options.comment = null; - }else { - if(typeof options.comment === 'string'){ - options.comment = Buffer.from(options.comment, options.encoding); - } - if(!isBuffer$1(options.comment)){ - throw new CsvError$1('CSV_INVALID_OPTION_COMMENT', [ - 'Invalid option comment:', - 'comment must be a buffer or a string,', - `got ${JSON.stringify(options.comment)}` - ], options); - } + if(typeof delimiter === 'string'){ + delimiter = Buffer.from(delimiter, options.encoding); } - // Normalize option `delimiter` - const delimiter_json = JSON.stringify(options.delimiter); - if(!Array.isArray(options.delimiter)) options.delimiter = [options.delimiter]; - if(options.delimiter.length === 0){ + if(!isBuffer$1(delimiter) || delimiter.length === 0){ throw new CsvError$1('CSV_INVALID_OPTION_DELIMITER', [ 'Invalid option delimiter:', 'delimiter must be a non empty string or buffer or array of string|buffer,', `got ${delimiter_json}` ], options); } - options.delimiter = options.delimiter.map(function(delimiter){ - if(delimiter === undefined || delimiter === null || delimiter === false){ - return Buffer.from(',', options.encoding); - } - if(typeof delimiter === 'string'){ - delimiter = Buffer.from(delimiter, options.encoding); - } - if(!isBuffer$1(delimiter) || delimiter.length === 0){ - throw new CsvError$1('CSV_INVALID_OPTION_DELIMITER', [ - 'Invalid option delimiter:', - 'delimiter must be a non empty string or buffer or array of string|buffer,', - `got ${delimiter_json}` - ], options); - } - return delimiter; - }); - // Normalize option `escape` - if(options.escape === undefined || options.escape === true){ - options.escape = Buffer.from('"', options.encoding); - }else if(typeof options.escape === 'string'){ - options.escape = Buffer.from(options.escape, options.encoding); - }else if (options.escape === null || options.escape === false){ - options.escape = null; - } - if(options.escape !== null){ - if(!isBuffer$1(options.escape)){ - throw new Error(`Invalid Option: escape must be a buffer, a string or a boolean, got ${JSON.stringify(options.escape)}`); - } + return delimiter; + }); + // Normalize option `escape` + if(options.escape === undefined || options.escape === true){ + options.escape = Buffer.from('"', options.encoding); + }else if(typeof options.escape === 'string'){ + options.escape = Buffer.from(options.escape, options.encoding); + }else if (options.escape === null || options.escape === false){ + options.escape = null; + } + if(options.escape !== null){ + if(!isBuffer$1(options.escape)){ + throw new Error(`Invalid Option: escape must be a buffer, a string or a boolean, got ${JSON.stringify(options.escape)}`); + } + } + // Normalize option `from` + if(options.from === undefined || options.from === null){ + options.from = 1; + }else { + if(typeof options.from === 'string' && /\d+/.test(options.from)){ + options.from = parseInt(options.from); } - // Normalize option `from` - if(options.from === undefined || options.from === null){ - options.from = 1; - }else { - if(typeof options.from === 'string' && /\d+/.test(options.from)){ - options.from = parseInt(options.from); - } - if(Number.isInteger(options.from)){ - if(options.from < 0){ - throw new Error(`Invalid Option: from must be a positive integer, got ${JSON.stringify(opts.from)}`); - } - }else { - throw new Error(`Invalid Option: from must be an integer, got ${JSON.stringify(options.from)}`); + if(Number.isInteger(options.from)){ + if(options.from < 0){ + throw new Error(`Invalid Option: from must be a positive integer, got ${JSON.stringify(opts.from)}`); } - } - // Normalize option `from_line` - if(options.from_line === undefined || options.from_line === null){ - options.from_line = 1; }else { - if(typeof options.from_line === 'string' && /\d+/.test(options.from_line)){ - options.from_line = parseInt(options.from_line); - } - if(Number.isInteger(options.from_line)){ - if(options.from_line <= 0){ - throw new Error(`Invalid Option: from_line must be a positive integer greater than 0, got ${JSON.stringify(opts.from_line)}`); - } - }else { - throw new Error(`Invalid Option: from_line must be an integer, got ${JSON.stringify(opts.from_line)}`); - } - } - // Normalize options `ignore_last_delimiters` - if(options.ignore_last_delimiters === undefined || options.ignore_last_delimiters === null){ - options.ignore_last_delimiters = false; - }else if(typeof options.ignore_last_delimiters === 'number'){ - options.ignore_last_delimiters = Math.floor(options.ignore_last_delimiters); - if(options.ignore_last_delimiters === 0){ - options.ignore_last_delimiters = false; - } - }else if(typeof options.ignore_last_delimiters !== 'boolean'){ - throw new CsvError$1('CSV_INVALID_OPTION_IGNORE_LAST_DELIMITERS', [ - 'Invalid option `ignore_last_delimiters`:', - 'the value must be a boolean value or an integer,', - `got ${JSON.stringify(options.ignore_last_delimiters)}` - ], options); + throw new Error(`Invalid Option: from must be an integer, got ${JSON.stringify(options.from)}`); } - if(options.ignore_last_delimiters === true && options.columns === false){ - throw new CsvError$1('CSV_IGNORE_LAST_DELIMITERS_REQUIRES_COLUMNS', [ - 'The option `ignore_last_delimiters`', - 'requires the activation of the `columns` option' - ], options); + } + // Normalize option `from_line` + if(options.from_line === undefined || options.from_line === null){ + options.from_line = 1; + }else { + if(typeof options.from_line === 'string' && /\d+/.test(options.from_line)){ + options.from_line = parseInt(options.from_line); } - // Normalize option `info` - if(options.info === undefined || options.info === null || options.info === false){ - options.info = false; - }else if(options.info !== true){ - throw new Error(`Invalid Option: info must be true, got ${JSON.stringify(options.info)}`); - } - // Normalize option `max_record_size` - if(options.max_record_size === undefined || options.max_record_size === null || options.max_record_size === false){ - options.max_record_size = 0; - }else if(Number.isInteger(options.max_record_size) && options.max_record_size >= 0);else if(typeof options.max_record_size === 'string' && /\d+/.test(options.max_record_size)){ - options.max_record_size = parseInt(options.max_record_size); - }else { - throw new Error(`Invalid Option: max_record_size must be a positive integer, got ${JSON.stringify(options.max_record_size)}`); - } - // Normalize option `objname` - if(options.objname === undefined || options.objname === null || options.objname === false){ - options.objname = undefined; - }else if(isBuffer$1(options.objname)){ - if(options.objname.length === 0){ - throw new Error(`Invalid Option: objname must be a non empty buffer`); - } - if(options.encoding === null);else { - options.objname = options.objname.toString(options.encoding); - } - }else if(typeof options.objname === 'string'){ - if(options.objname.length === 0){ - throw new Error(`Invalid Option: objname must be a non empty string`); - } - // Great, nothing to do - }else if(typeof options.objname === 'number');else { - throw new Error(`Invalid Option: objname must be a string or a buffer, got ${options.objname}`); - } - if(options.objname !== undefined){ - if(typeof options.objname === 'number'){ - if(options.columns !== false){ - throw Error('Invalid Option: objname index cannot be combined with columns or be defined as a field'); - } - }else { // A string or a buffer - if(options.columns === false){ - throw Error('Invalid Option: objname field must be combined with columns or be defined as an index'); - } + if(Number.isInteger(options.from_line)){ + if(options.from_line <= 0){ + throw new Error(`Invalid Option: from_line must be a positive integer greater than 0, got ${JSON.stringify(opts.from_line)}`); } + }else { + throw new Error(`Invalid Option: from_line must be an integer, got ${JSON.stringify(opts.from_line)}`); } - // Normalize option `on_record` - if(options.on_record === undefined || options.on_record === null){ - options.on_record = undefined; - }else if(typeof options.on_record !== 'function'){ - throw new CsvError$1('CSV_INVALID_OPTION_ON_RECORD', [ - 'Invalid option `on_record`:', - 'expect a function,', - `got ${JSON.stringify(options.on_record)}` - ], options); + } + // Normalize options `ignore_last_delimiters` + if(options.ignore_last_delimiters === undefined || options.ignore_last_delimiters === null){ + options.ignore_last_delimiters = false; + }else if(typeof options.ignore_last_delimiters === 'number'){ + options.ignore_last_delimiters = Math.floor(options.ignore_last_delimiters); + if(options.ignore_last_delimiters === 0){ + options.ignore_last_delimiters = false; } - // Normalize option `quote` - if(options.quote === null || options.quote === false || options.quote === ''){ - options.quote = null; - }else { - if(options.quote === undefined || options.quote === true){ - options.quote = Buffer.from('"', options.encoding); - }else if(typeof options.quote === 'string'){ - options.quote = Buffer.from(options.quote, options.encoding); + }else if(typeof options.ignore_last_delimiters !== 'boolean'){ + throw new CsvError$1('CSV_INVALID_OPTION_IGNORE_LAST_DELIMITERS', [ + 'Invalid option `ignore_last_delimiters`:', + 'the value must be a boolean value or an integer,', + `got ${JSON.stringify(options.ignore_last_delimiters)}` + ], options); + } + if(options.ignore_last_delimiters === true && options.columns === false){ + throw new CsvError$1('CSV_IGNORE_LAST_DELIMITERS_REQUIRES_COLUMNS', [ + 'The option `ignore_last_delimiters`', + 'requires the activation of the `columns` option' + ], options); + } + // Normalize option `info` + if(options.info === undefined || options.info === null || options.info === false){ + options.info = false; + }else if(options.info !== true){ + throw new Error(`Invalid Option: info must be true, got ${JSON.stringify(options.info)}`); + } + // Normalize option `max_record_size` + if(options.max_record_size === undefined || options.max_record_size === null || options.max_record_size === false){ + options.max_record_size = 0; + }else if(Number.isInteger(options.max_record_size) && options.max_record_size >= 0);else if(typeof options.max_record_size === 'string' && /\d+/.test(options.max_record_size)){ + options.max_record_size = parseInt(options.max_record_size); + }else { + throw new Error(`Invalid Option: max_record_size must be a positive integer, got ${JSON.stringify(options.max_record_size)}`); + } + // Normalize option `objname` + if(options.objname === undefined || options.objname === null || options.objname === false){ + options.objname = undefined; + }else if(isBuffer$1(options.objname)){ + if(options.objname.length === 0){ + throw new Error(`Invalid Option: objname must be a non empty buffer`); + } + if(options.encoding === null);else { + options.objname = options.objname.toString(options.encoding); + } + }else if(typeof options.objname === 'string'){ + if(options.objname.length === 0){ + throw new Error(`Invalid Option: objname must be a non empty string`); + } + // Great, nothing to do + }else if(typeof options.objname === 'number');else { + throw new Error(`Invalid Option: objname must be a string or a buffer, got ${options.objname}`); + } + if(options.objname !== undefined){ + if(typeof options.objname === 'number'){ + if(options.columns !== false){ + throw Error('Invalid Option: objname index cannot be combined with columns or be defined as a field'); } - if(!isBuffer$1(options.quote)){ - throw new Error(`Invalid Option: quote must be a buffer or a string, got ${JSON.stringify(options.quote)}`); + }else { // A string or a buffer + if(options.columns === false){ + throw Error('Invalid Option: objname field must be combined with columns or be defined as an index'); } } - // Normalize option `raw` - if(options.raw === undefined || options.raw === null || options.raw === false){ - options.raw = false; - }else if(options.raw !== true){ - throw new Error(`Invalid Option: raw must be true, got ${JSON.stringify(options.raw)}`); - } - // Normalize option `record_delimiter` - if(options.record_delimiter === undefined){ - options.record_delimiter = []; - }else if(typeof options.record_delimiter === 'string' || isBuffer$1(options.record_delimiter)){ - if(options.record_delimiter.length === 0){ - throw new CsvError$1('CSV_INVALID_OPTION_RECORD_DELIMITER', [ - 'Invalid option `record_delimiter`:', - 'value must be a non empty string or buffer,', - `got ${JSON.stringify(options.record_delimiter)}` - ], options); - } - options.record_delimiter = [options.record_delimiter]; - }else if(!Array.isArray(options.record_delimiter)){ + } + // Normalize option `on_record` + if(options.on_record === undefined || options.on_record === null){ + options.on_record = undefined; + }else if(typeof options.on_record !== 'function'){ + throw new CsvError$1('CSV_INVALID_OPTION_ON_RECORD', [ + 'Invalid option `on_record`:', + 'expect a function,', + `got ${JSON.stringify(options.on_record)}` + ], options); + } + // Normalize option `quote` + if(options.quote === null || options.quote === false || options.quote === ''){ + options.quote = null; + }else { + if(options.quote === undefined || options.quote === true){ + options.quote = Buffer.from('"', options.encoding); + }else if(typeof options.quote === 'string'){ + options.quote = Buffer.from(options.quote, options.encoding); + } + if(!isBuffer$1(options.quote)){ + throw new Error(`Invalid Option: quote must be a buffer or a string, got ${JSON.stringify(options.quote)}`); + } + } + // Normalize option `raw` + if(options.raw === undefined || options.raw === null || options.raw === false){ + options.raw = false; + }else if(options.raw !== true){ + throw new Error(`Invalid Option: raw must be true, got ${JSON.stringify(options.raw)}`); + } + // Normalize option `record_delimiter` + if(options.record_delimiter === undefined){ + options.record_delimiter = []; + }else if(typeof options.record_delimiter === 'string' || isBuffer$1(options.record_delimiter)){ + if(options.record_delimiter.length === 0){ throw new CsvError$1('CSV_INVALID_OPTION_RECORD_DELIMITER', [ 'Invalid option `record_delimiter`:', - 'value must be a string, a buffer or array of string|buffer,', + 'value must be a non empty string or buffer,', `got ${JSON.stringify(options.record_delimiter)}` ], options); } - options.record_delimiter = options.record_delimiter.map(function(rd, i){ - if(typeof rd !== 'string' && ! isBuffer$1(rd)){ - throw new CsvError$1('CSV_INVALID_OPTION_RECORD_DELIMITER', [ - 'Invalid option `record_delimiter`:', - 'value must be a string, a buffer or array of string|buffer', - `at index ${i},`, - `got ${JSON.stringify(rd)}` - ], options); - }else if(rd.length === 0){ - throw new CsvError$1('CSV_INVALID_OPTION_RECORD_DELIMITER', [ - 'Invalid option `record_delimiter`:', - 'value must be a non empty string or buffer', - `at index ${i},`, - `got ${JSON.stringify(rd)}` - ], options); - } - if(typeof rd === 'string'){ - rd = Buffer.from(rd, options.encoding); - } - return rd; - }); - // Normalize option `relax_column_count` - if(typeof options.relax_column_count === 'boolean');else if(options.relax_column_count === undefined || options.relax_column_count === null){ - options.relax_column_count = false; - }else { - throw new Error(`Invalid Option: relax_column_count must be a boolean, got ${JSON.stringify(options.relax_column_count)}`); - } - if(typeof options.relax_column_count_less === 'boolean');else if(options.relax_column_count_less === undefined || options.relax_column_count_less === null){ - options.relax_column_count_less = false; - }else { - throw new Error(`Invalid Option: relax_column_count_less must be a boolean, got ${JSON.stringify(options.relax_column_count_less)}`); - } - if(typeof options.relax_column_count_more === 'boolean');else if(options.relax_column_count_more === undefined || options.relax_column_count_more === null){ - options.relax_column_count_more = false; - }else { - throw new Error(`Invalid Option: relax_column_count_more must be a boolean, got ${JSON.stringify(options.relax_column_count_more)}`); - } - // Normalize option `relax_quotes` - if(typeof options.relax_quotes === 'boolean');else if(options.relax_quotes === undefined || options.relax_quotes === null){ - options.relax_quotes = false; - }else { - throw new Error(`Invalid Option: relax_quotes must be a boolean, got ${JSON.stringify(options.relax_quotes)}`); + options.record_delimiter = [options.record_delimiter]; + }else if(!Array.isArray(options.record_delimiter)){ + throw new CsvError$1('CSV_INVALID_OPTION_RECORD_DELIMITER', [ + 'Invalid option `record_delimiter`:', + 'value must be a string, a buffer or array of string|buffer,', + `got ${JSON.stringify(options.record_delimiter)}` + ], options); + } + options.record_delimiter = options.record_delimiter.map(function(rd, i){ + if(typeof rd !== 'string' && ! isBuffer$1(rd)){ + throw new CsvError$1('CSV_INVALID_OPTION_RECORD_DELIMITER', [ + 'Invalid option `record_delimiter`:', + 'value must be a string, a buffer or array of string|buffer', + `at index ${i},`, + `got ${JSON.stringify(rd)}` + ], options); + }else if(rd.length === 0){ + throw new CsvError$1('CSV_INVALID_OPTION_RECORD_DELIMITER', [ + 'Invalid option `record_delimiter`:', + 'value must be a non empty string or buffer', + `at index ${i},`, + `got ${JSON.stringify(rd)}` + ], options); } - // Normalize option `skip_empty_lines` - if(typeof options.skip_empty_lines === 'boolean');else if(options.skip_empty_lines === undefined || options.skip_empty_lines === null){ - options.skip_empty_lines = false; - }else { - throw new Error(`Invalid Option: skip_empty_lines must be a boolean, got ${JSON.stringify(options.skip_empty_lines)}`); + if(typeof rd === 'string'){ + rd = Buffer.from(rd, options.encoding); } - // Normalize option `skip_records_with_empty_values` - if(typeof options.skip_records_with_empty_values === 'boolean');else if(options.skip_records_with_empty_values === undefined || options.skip_records_with_empty_values === null){ - options.skip_records_with_empty_values = false; - }else { - throw new Error(`Invalid Option: skip_records_with_empty_values must be a boolean, got ${JSON.stringify(options.skip_records_with_empty_values)}`); + return rd; + }); + // Normalize option `relax_column_count` + if(typeof options.relax_column_count === 'boolean');else if(options.relax_column_count === undefined || options.relax_column_count === null){ + options.relax_column_count = false; + }else { + throw new Error(`Invalid Option: relax_column_count must be a boolean, got ${JSON.stringify(options.relax_column_count)}`); + } + if(typeof options.relax_column_count_less === 'boolean');else if(options.relax_column_count_less === undefined || options.relax_column_count_less === null){ + options.relax_column_count_less = false; + }else { + throw new Error(`Invalid Option: relax_column_count_less must be a boolean, got ${JSON.stringify(options.relax_column_count_less)}`); + } + if(typeof options.relax_column_count_more === 'boolean');else if(options.relax_column_count_more === undefined || options.relax_column_count_more === null){ + options.relax_column_count_more = false; + }else { + throw new Error(`Invalid Option: relax_column_count_more must be a boolean, got ${JSON.stringify(options.relax_column_count_more)}`); + } + // Normalize option `relax_quotes` + if(typeof options.relax_quotes === 'boolean');else if(options.relax_quotes === undefined || options.relax_quotes === null){ + options.relax_quotes = false; + }else { + throw new Error(`Invalid Option: relax_quotes must be a boolean, got ${JSON.stringify(options.relax_quotes)}`); + } + // Normalize option `skip_empty_lines` + if(typeof options.skip_empty_lines === 'boolean');else if(options.skip_empty_lines === undefined || options.skip_empty_lines === null){ + options.skip_empty_lines = false; + }else { + throw new Error(`Invalid Option: skip_empty_lines must be a boolean, got ${JSON.stringify(options.skip_empty_lines)}`); + } + // Normalize option `skip_records_with_empty_values` + if(typeof options.skip_records_with_empty_values === 'boolean');else if(options.skip_records_with_empty_values === undefined || options.skip_records_with_empty_values === null){ + options.skip_records_with_empty_values = false; + }else { + throw new Error(`Invalid Option: skip_records_with_empty_values must be a boolean, got ${JSON.stringify(options.skip_records_with_empty_values)}`); + } + // Normalize option `skip_records_with_error` + if(typeof options.skip_records_with_error === 'boolean');else if(options.skip_records_with_error === undefined || options.skip_records_with_error === null){ + options.skip_records_with_error = false; + }else { + throw new Error(`Invalid Option: skip_records_with_error must be a boolean, got ${JSON.stringify(options.skip_records_with_error)}`); + } + // Normalize option `rtrim` + if(options.rtrim === undefined || options.rtrim === null || options.rtrim === false){ + options.rtrim = false; + }else if(options.rtrim !== true){ + throw new Error(`Invalid Option: rtrim must be a boolean, got ${JSON.stringify(options.rtrim)}`); + } + // Normalize option `ltrim` + if(options.ltrim === undefined || options.ltrim === null || options.ltrim === false){ + options.ltrim = false; + }else if(options.ltrim !== true){ + throw new Error(`Invalid Option: ltrim must be a boolean, got ${JSON.stringify(options.ltrim)}`); + } + // Normalize option `trim` + if(options.trim === undefined || options.trim === null || options.trim === false){ + options.trim = false; + }else if(options.trim !== true){ + throw new Error(`Invalid Option: trim must be a boolean, got ${JSON.stringify(options.trim)}`); + } + // Normalize options `trim`, `ltrim` and `rtrim` + if(options.trim === true && opts.ltrim !== false){ + options.ltrim = true; + }else if(options.ltrim !== true){ + options.ltrim = false; + } + if(options.trim === true && opts.rtrim !== false){ + options.rtrim = true; + }else if(options.rtrim !== true){ + options.rtrim = false; + } + // Normalize option `to` + if(options.to === undefined || options.to === null){ + options.to = -1; + }else { + if(typeof options.to === 'string' && /\d+/.test(options.to)){ + options.to = parseInt(options.to); } - // Normalize option `skip_records_with_error` - if(typeof options.skip_records_with_error === 'boolean');else if(options.skip_records_with_error === undefined || options.skip_records_with_error === null){ - options.skip_records_with_error = false; - }else { - throw new Error(`Invalid Option: skip_records_with_error must be a boolean, got ${JSON.stringify(options.skip_records_with_error)}`); - } - // Normalize option `rtrim` - if(options.rtrim === undefined || options.rtrim === null || options.rtrim === false){ - options.rtrim = false; - }else if(options.rtrim !== true){ - throw new Error(`Invalid Option: rtrim must be a boolean, got ${JSON.stringify(options.rtrim)}`); - } - // Normalize option `ltrim` - if(options.ltrim === undefined || options.ltrim === null || options.ltrim === false){ - options.ltrim = false; - }else if(options.ltrim !== true){ - throw new Error(`Invalid Option: ltrim must be a boolean, got ${JSON.stringify(options.ltrim)}`); - } - // Normalize option `trim` - if(options.trim === undefined || options.trim === null || options.trim === false){ - options.trim = false; - }else if(options.trim !== true){ - throw new Error(`Invalid Option: trim must be a boolean, got ${JSON.stringify(options.trim)}`); - } - // Normalize options `trim`, `ltrim` and `rtrim` - if(options.trim === true && opts.ltrim !== false){ - options.ltrim = true; - }else if(options.ltrim !== true){ - options.ltrim = false; - } - if(options.trim === true && opts.rtrim !== false){ - options.rtrim = true; - }else if(options.rtrim !== true){ - options.rtrim = false; - } - // Normalize option `to` - if(options.to === undefined || options.to === null){ - options.to = -1; - }else { - if(typeof options.to === 'string' && /\d+/.test(options.to)){ - options.to = parseInt(options.to); + if(Number.isInteger(options.to)){ + if(options.to <= 0){ + throw new Error(`Invalid Option: to must be a positive integer greater than 0, got ${JSON.stringify(opts.to)}`); } - if(Number.isInteger(options.to)){ - if(options.to <= 0){ - throw new Error(`Invalid Option: to must be a positive integer greater than 0, got ${JSON.stringify(opts.to)}`); - } - }else { - throw new Error(`Invalid Option: to must be an integer, got ${JSON.stringify(opts.to)}`); - } - } - // Normalize option `to_line` - if(options.to_line === undefined || options.to_line === null){ - options.to_line = -1; }else { - if(typeof options.to_line === 'string' && /\d+/.test(options.to_line)){ - options.to_line = parseInt(options.to_line); - } - if(Number.isInteger(options.to_line)){ - if(options.to_line <= 0){ - throw new Error(`Invalid Option: to_line must be a positive integer greater than 0, got ${JSON.stringify(opts.to_line)}`); - } - }else { - throw new Error(`Invalid Option: to_line must be an integer, got ${JSON.stringify(opts.to_line)}`); - } + throw new Error(`Invalid Option: to must be an integer, got ${JSON.stringify(opts.to)}`); } - this.info = { - bytes: 0, - comment_lines: 0, - empty_lines: 0, - invalid_field_length: 0, - lines: 1, - records: 0 - }; - this.options = options; - this.state = { - bomSkipped: false, - bufBytesStart: 0, - castField: fnCastField, - commenting: false, - // Current error encountered by a record - error: undefined, - enabled: options.from_line === 1, - escaping: false, - escapeIsQuote: isBuffer$1(options.escape) && isBuffer$1(options.quote) && Buffer.compare(options.escape, options.quote) === 0, - // columns can be `false`, `true`, `Array` - expectedRecordLength: Array.isArray(options.columns) ? options.columns.length : undefined, - field: new ResizeableBuffer(20), - firstLineToHeaders: fnFirstLineToHeaders, - needMoreDataSize: Math.max( - // Skip if the remaining buffer smaller than comment - options.comment !== null ? options.comment.length : 0, - // Skip if the remaining buffer can be delimiter - ...options.delimiter.map((delimiter) => delimiter.length), - // Skip if the remaining buffer can be escape sequence - options.quote !== null ? options.quote.length : 0, - ), - previousBuf: undefined, - quoting: false, - stop: false, - rawBuffer: new ResizeableBuffer(100), - record: [], - recordHasError: false, - record_length: 0, - recordDelimiterMaxLength: options.record_delimiter.length === 0 ? 2 : Math.max(...options.record_delimiter.map((v) => v.length)), - trimChars: [Buffer.from(' ', options.encoding)[0], Buffer.from('\t', options.encoding)[0]], - wasQuoting: false, - wasRowDelimiter: false - }; } - // Implementation of `Transform._transform` - _transform(buf, encoding, callback){ - if(this.state.stop === true){ - return; - } - const err = this.__parse(buf, false); - if(err !== undefined){ - this.state.stop = true; - } - callback(err); - } - // Implementation of `Transform._flush` - _flush(callback){ - if(this.state.stop === true){ - return; + // Normalize option `to_line` + if(options.to_line === undefined || options.to_line === null){ + options.to_line = -1; + }else { + if(typeof options.to_line === 'string' && /\d+/.test(options.to_line)){ + options.to_line = parseInt(options.to_line); } - const err = this.__parse(undefined, true); - callback(err); - } - // Central parser implementation - __parse(nextBuf, end){ - const {bom, comment, escape, from_line, ltrim, max_record_size, quote, raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options; - let {record_delimiter} = this.options; - const {bomSkipped, previousBuf, rawBuffer, escapeIsQuote} = this.state; - let buf; - if(previousBuf === undefined){ - if(nextBuf === undefined){ - // Handle empty string - this.push(null); - return; - }else { - buf = nextBuf; + if(Number.isInteger(options.to_line)){ + if(options.to_line <= 0){ + throw new Error(`Invalid Option: to_line must be a positive integer greater than 0, got ${JSON.stringify(opts.to_line)}`); } - }else if(previousBuf !== undefined && nextBuf === undefined){ - buf = previousBuf; }else { - buf = Buffer.concat([previousBuf, nextBuf]); - } - // Handle UTF BOM - if(bomSkipped === false){ - if(bom === false){ - this.state.bomSkipped = true; - }else if(buf.length < 3){ - // No enough data - if(end === false){ - // Wait for more data - this.state.previousBuf = buf; + throw new Error(`Invalid Option: to_line must be an integer, got ${JSON.stringify(opts.to_line)}`); + } + } + return options; + }; + + const isRecordEmpty = function(record){ + return record.every((field) => field == null || field.toString && field.toString().trim() === ''); + }; + + // white space characters + // https://en.wikipedia.org/wiki/Whitespace_character + // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions/Character_Classes#Types + // \f\n\r\t\v\u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff + const tab = 9; + const nl = 10; // \n, 0x0A in hexadecimal, 10 in decimal + const np = 12; + const cr = 13; // \r, 0x0D in hexadécimal, 13 in decimal + const space = 32; + const boms = { + // Note, the following are equals: + // Buffer.from("\ufeff") + // Buffer.from([239, 187, 191]) + // Buffer.from('EFBBBF', 'hex') + 'utf8': Buffer.from([239, 187, 191]), + // Note, the following are equals: + // Buffer.from "\ufeff", 'utf16le + // Buffer.from([255, 254]) + 'utf16le': Buffer.from([255, 254]) + }; + + const transform$1 = function(original_options = {}) { + const info = { + bytes: 0, + comment_lines: 0, + empty_lines: 0, + invalid_field_length: 0, + lines: 1, + records: 0 + }; + const options = normalize_options$1(original_options); + return { + info: info, + original_options: original_options, + options: options, + state: init_state(options), + __needMoreData: function(i, bufLen, end){ + if(end) return false; + const {quote} = this.options; + const {quoting, needMoreDataSize, recordDelimiterMaxLength} = this.state; + const numOfCharLeft = bufLen - i - 1; + const requiredLength = Math.max( + needMoreDataSize, + // Skip if the remaining buffer smaller than record delimiter + recordDelimiterMaxLength, + // Skip if the remaining buffer can be record delimiter following the closing quote + // 1 is for quote.length + quoting ? (quote.length + recordDelimiterMaxLength) : 0, + ); + return numOfCharLeft < requiredLength; + }, + // Central parser implementation + parse: function(nextBuf, end, push, close){ + const {bom, comment, escape, from_line, ltrim, max_record_size, quote, raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options; + let {record_delimiter} = this.options; + const {bomSkipped, previousBuf, rawBuffer, escapeIsQuote} = this.state; + let buf; + if(previousBuf === undefined){ + if(nextBuf === undefined){ + // Handle empty string + close(); return; + }else { + buf = nextBuf; } + }else if(previousBuf !== undefined && nextBuf === undefined){ + buf = previousBuf; }else { - for(const encoding in boms){ - if(boms[encoding].compare(buf, 0, boms[encoding].length) === 0){ - // Skip BOM - const bomLength = boms[encoding].length; - this.state.bufBytesStart += bomLength; - buf = buf.slice(bomLength); - // Renormalize original options with the new encoding - this.__normalizeOptions({...this.__originalOptions, encoding: encoding}); - break; + buf = Buffer.concat([previousBuf, nextBuf]); + } + // Handle UTF BOM + if(bomSkipped === false){ + if(bom === false){ + this.state.bomSkipped = true; + }else if(buf.length < 3){ + // No enough data + if(end === false){ + // Wait for more data + this.state.previousBuf = buf; + return; + } + }else { + for(const encoding in boms){ + if(boms[encoding].compare(buf, 0, boms[encoding].length) === 0){ + // Skip BOM + const bomLength = boms[encoding].length; + this.state.bufBytesStart += bomLength; + buf = buf.slice(bomLength); + // Renormalize original options with the new encoding + this.options = normalize_options$1({...this.original_options, encoding: encoding}); + break; + } } + this.state.bomSkipped = true; } - this.state.bomSkipped = true; } - } - const bufLen = buf.length; - let pos; - for(pos = 0; pos < bufLen; pos++){ - // Ensure we get enough space to look ahead - // There should be a way to move this out of the loop - if(this.__needMoreData(pos, bufLen, end)){ - break; - } - if(this.state.wasRowDelimiter === true){ - this.info.lines++; - this.state.wasRowDelimiter = false; - } - if(to_line !== -1 && this.info.lines > to_line){ - this.state.stop = true; - this.push(null); - return; - } - // Auto discovery of record_delimiter, unix, mac and windows supported - if(this.state.quoting === false && record_delimiter.length === 0){ - const record_delimiterCount = this.__autoDiscoverRecordDelimiter(buf, pos); - if(record_delimiterCount){ - record_delimiter = this.options.record_delimiter; + const bufLen = buf.length; + let pos; + for(pos = 0; pos < bufLen; pos++){ + // Ensure we get enough space to look ahead + // There should be a way to move this out of the loop + if(this.__needMoreData(pos, bufLen, end)){ + break; } - } - const chr = buf[pos]; - if(raw === true){ - rawBuffer.append(chr); - } - if((chr === cr || chr === nl) && this.state.wasRowDelimiter === false){ - this.state.wasRowDelimiter = true; - } - // Previous char was a valid escape char - // treat the current char as a regular char - if(this.state.escaping === true){ - this.state.escaping = false; - }else { - // Escape is only active inside quoted fields - // We are quoting, the char is an escape chr and there is a chr to escape - // if(escape !== null && this.state.quoting === true && chr === escape && pos + 1 < bufLen){ - if(escape !== null && this.state.quoting === true && this.__isEscape(buf, pos, chr) && pos + escape.length < bufLen){ - if(escapeIsQuote){ - if(this.__isQuote(buf, pos+escape.length)){ - this.state.escaping = true; - pos += escape.length - 1; - continue; - } - }else { - this.state.escaping = true; - pos += escape.length - 1; - continue; + if(this.state.wasRowDelimiter === true){ + this.info.lines++; + this.state.wasRowDelimiter = false; + } + if(to_line !== -1 && this.info.lines > to_line){ + this.state.stop = true; + close(); + return; + } + // Auto discovery of record_delimiter, unix, mac and windows supported + if(this.state.quoting === false && record_delimiter.length === 0){ + const record_delimiterCount = this.__autoDiscoverRecordDelimiter(buf, pos); + if(record_delimiterCount){ + record_delimiter = this.options.record_delimiter; } } - // Not currently escaping and chr is a quote - // TODO: need to compare bytes instead of single char - if(this.state.commenting === false && this.__isQuote(buf, pos)){ - if(this.state.quoting === true){ - const nextChr = buf[pos+quote.length]; - const isNextChrTrimable = rtrim && this.__isCharTrimable(nextChr); - const isNextChrComment = comment !== null && this.__compareBytes(comment, buf, pos+quote.length, nextChr); - const isNextChrDelimiter = this.__isDelimiter(buf, pos+quote.length, nextChr); - const isNextChrRecordDelimiter = record_delimiter.length === 0 ? this.__autoDiscoverRecordDelimiter(buf, pos+quote.length) : this.__isRecordDelimiter(nextChr, buf, pos+quote.length); - // Escape a quote - // Treat next char as a regular character - if(escape !== null && this.__isEscape(buf, pos, chr) && this.__isQuote(buf, pos + escape.length)){ + const chr = buf[pos]; + if(raw === true){ + rawBuffer.append(chr); + } + if((chr === cr || chr === nl) && this.state.wasRowDelimiter === false){ + this.state.wasRowDelimiter = true; + } + // Previous char was a valid escape char + // treat the current char as a regular char + if(this.state.escaping === true){ + this.state.escaping = false; + }else { + // Escape is only active inside quoted fields + // We are quoting, the char is an escape chr and there is a chr to escape + // if(escape !== null && this.state.quoting === true && chr === escape && pos + 1 < bufLen){ + if(escape !== null && this.state.quoting === true && this.__isEscape(buf, pos, chr) && pos + escape.length < bufLen){ + if(escapeIsQuote){ + if(this.__isQuote(buf, pos+escape.length)){ + this.state.escaping = true; + pos += escape.length - 1; + continue; + } + }else { + this.state.escaping = true; pos += escape.length - 1; - }else if(!nextChr || isNextChrDelimiter || isNextChrRecordDelimiter || isNextChrComment || isNextChrTrimable){ - this.state.quoting = false; - this.state.wasQuoting = true; - pos += quote.length - 1; continue; - }else if(relax_quotes === false){ - const err = this.__error( - new CsvError$1('CSV_INVALID_CLOSING_QUOTE', [ - 'Invalid Closing Quote:', - `got "${String.fromCharCode(nextChr)}"`, - `at line ${this.info.lines}`, - 'instead of delimiter, record delimiter, trimable character', - '(if activated) or comment', - ], this.options, this.__infoField()) - ); - if(err !== undefined) return err; - }else { - this.state.quoting = false; - this.state.wasQuoting = true; - this.state.field.prepend(quote); - pos += quote.length - 1; } - }else { - if(this.state.field.length !== 0){ - // In relax_quotes mode, treat opening quote preceded by chrs as regular - if(relax_quotes === false){ + } + // Not currently escaping and chr is a quote + // TODO: need to compare bytes instead of single char + if(this.state.commenting === false && this.__isQuote(buf, pos)){ + if(this.state.quoting === true){ + const nextChr = buf[pos+quote.length]; + const isNextChrTrimable = rtrim && this.__isCharTrimable(nextChr); + const isNextChrComment = comment !== null && this.__compareBytes(comment, buf, pos+quote.length, nextChr); + const isNextChrDelimiter = this.__isDelimiter(buf, pos+quote.length, nextChr); + const isNextChrRecordDelimiter = record_delimiter.length === 0 ? this.__autoDiscoverRecordDelimiter(buf, pos+quote.length) : this.__isRecordDelimiter(nextChr, buf, pos+quote.length); + // Escape a quote + // Treat next char as a regular character + if(escape !== null && this.__isEscape(buf, pos, chr) && this.__isQuote(buf, pos + escape.length)){ + pos += escape.length - 1; + }else if(!nextChr || isNextChrDelimiter || isNextChrRecordDelimiter || isNextChrComment || isNextChrTrimable){ + this.state.quoting = false; + this.state.wasQuoting = true; + pos += quote.length - 1; + continue; + }else if(relax_quotes === false){ const err = this.__error( - new CsvError$1('INVALID_OPENING_QUOTE', [ - 'Invalid Opening Quote:', - `a quote is found inside a field at line ${this.info.lines}`, - ], this.options, this.__infoField(), { - field: this.state.field, - }) + new CsvError$1('CSV_INVALID_CLOSING_QUOTE', [ + 'Invalid Closing Quote:', + `got "${String.fromCharCode(nextChr)}"`, + `at line ${this.info.lines}`, + 'instead of delimiter, record delimiter, trimable character', + '(if activated) or comment', + ], this.options, this.__infoField()) ); if(err !== undefined) return err; + }else { + this.state.quoting = false; + this.state.wasQuoting = true; + this.state.field.prepend(quote); + pos += quote.length - 1; } }else { - this.state.quoting = true; - pos += quote.length - 1; - continue; - } - } - } - if(this.state.quoting === false){ - const recordDelimiterLength = this.__isRecordDelimiter(chr, buf, pos); - if(recordDelimiterLength !== 0){ - // Do not emit comments which take a full line - const skipCommentLine = this.state.commenting && (this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0); - if(skipCommentLine){ - this.info.comment_lines++; - // Skip full comment line - }else { - // Activate records emition if above from_line - if(this.state.enabled === false && this.info.lines + (this.state.wasRowDelimiter === true ? 1: 0) >= from_line){ - this.state.enabled = true; - this.__resetField(); - this.__resetRecord(); - pos += recordDelimiterLength - 1; + if(this.state.field.length !== 0){ + // In relax_quotes mode, treat opening quote preceded by chrs as regular + if(relax_quotes === false){ + const err = this.__error( + new CsvError$1('INVALID_OPENING_QUOTE', [ + 'Invalid Opening Quote:', + `a quote is found inside a field at line ${this.info.lines}`, + ], this.options, this.__infoField(), { + field: this.state.field, + }) + ); + if(err !== undefined) return err; + } + }else { + this.state.quoting = true; + pos += quote.length - 1; continue; } - // Skip if line is empty and skip_empty_lines activated - if(skip_empty_lines === true && this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0){ - this.info.empty_lines++; - pos += recordDelimiterLength - 1; - continue; + } + } + if(this.state.quoting === false){ + const recordDelimiterLength = this.__isRecordDelimiter(chr, buf, pos); + if(recordDelimiterLength !== 0){ + // Do not emit comments which take a full line + const skipCommentLine = this.state.commenting && (this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0); + if(skipCommentLine){ + this.info.comment_lines++; + // Skip full comment line + }else { + // Activate records emition if above from_line + if(this.state.enabled === false && this.info.lines + (this.state.wasRowDelimiter === true ? 1: 0) >= from_line){ + this.state.enabled = true; + this.__resetField(); + this.__resetRecord(); + pos += recordDelimiterLength - 1; + continue; + } + // Skip if line is empty and skip_empty_lines activated + if(skip_empty_lines === true && this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0){ + this.info.empty_lines++; + pos += recordDelimiterLength - 1; + continue; + } + this.info.bytes = this.state.bufBytesStart + pos; + const errField = this.__onField(); + if(errField !== undefined) return errField; + this.info.bytes = this.state.bufBytesStart + pos + recordDelimiterLength; + const errRecord = this.__onRecord(push); + if(errRecord !== undefined) return errRecord; + if(to !== -1 && this.info.records >= to){ + this.state.stop = true; + close(); + return; + } } + this.state.commenting = false; + pos += recordDelimiterLength - 1; + continue; + } + if(this.state.commenting){ + continue; + } + const commentCount = comment === null ? 0 : this.__compareBytes(comment, buf, pos, chr); + if(commentCount !== 0){ + this.state.commenting = true; + continue; + } + const delimiterLength = this.__isDelimiter(buf, pos, chr); + if(delimiterLength !== 0){ this.info.bytes = this.state.bufBytesStart + pos; const errField = this.__onField(); if(errField !== undefined) return errField; - this.info.bytes = this.state.bufBytesStart + pos + recordDelimiterLength; - const errRecord = this.__onRecord(); - if(errRecord !== undefined) return errRecord; - if(to !== -1 && this.info.records >= to){ - this.state.stop = true; - this.push(null); - return; - } + pos += delimiterLength - 1; + continue; } - this.state.commenting = false; - pos += recordDelimiterLength - 1; - continue; } - if(this.state.commenting){ - continue; - } - const commentCount = comment === null ? 0 : this.__compareBytes(comment, buf, pos, chr); - if(commentCount !== 0){ - this.state.commenting = true; - continue; - } - const delimiterLength = this.__isDelimiter(buf, pos, chr); - if(delimiterLength !== 0){ - this.info.bytes = this.state.bufBytesStart + pos; - const errField = this.__onField(); - if(errField !== undefined) return errField; - pos += delimiterLength - 1; - continue; + } + if(this.state.commenting === false){ + if(max_record_size !== 0 && this.state.record_length + this.state.field.length > max_record_size){ + const err = this.__error( + new CsvError$1('CSV_MAX_RECORD_SIZE', [ + 'Max Record Size:', + 'record exceed the maximum number of tolerated bytes', + `of ${max_record_size}`, + `at line ${this.info.lines}`, + ], this.options, this.__infoField()) + ); + if(err !== undefined) return err; } } - } - if(this.state.commenting === false){ - if(max_record_size !== 0 && this.state.record_length + this.state.field.length > max_record_size){ + const lappend = ltrim === false || this.state.quoting === true || this.state.field.length !== 0 || !this.__isCharTrimable(chr); + // rtrim in non quoting is handle in __onField + const rappend = rtrim === false || this.state.wasQuoting === false; + if(lappend === true && rappend === true){ + this.state.field.append(chr); + }else if(rtrim === true && !this.__isCharTrimable(chr)){ const err = this.__error( - new CsvError$1('CSV_MAX_RECORD_SIZE', [ - 'Max Record Size:', - 'record exceed the maximum number of tolerated bytes', - `of ${max_record_size}`, + new CsvError$1('CSV_NON_TRIMABLE_CHAR_AFTER_CLOSING_QUOTE', [ + 'Invalid Closing Quote:', + 'found non trimable byte after quote', `at line ${this.info.lines}`, ], this.options, this.__infoField()) ); if(err !== undefined) return err; } } - const lappend = ltrim === false || this.state.quoting === true || this.state.field.length !== 0 || !this.__isCharTrimable(chr); - // rtrim in non quoting is handle in __onField - const rappend = rtrim === false || this.state.wasQuoting === false; - if(lappend === true && rappend === true){ - this.state.field.append(chr); - }else if(rtrim === true && !this.__isCharTrimable(chr)){ - const err = this.__error( - new CsvError$1('CSV_NON_TRIMABLE_CHAR_AFTER_CLOSING_QUOTE', [ - 'Invalid Closing Quote:', - 'found non trimable byte after quote', - `at line ${this.info.lines}`, - ], this.options, this.__infoField()) - ); - if(err !== undefined) return err; - } - } - if(end === true){ - // Ensure we are not ending in a quoting state - if(this.state.quoting === true){ - const err = this.__error( - new CsvError$1('CSV_QUOTE_NOT_CLOSED', [ - 'Quote Not Closed:', - `the parsing is finished with an opening quote at line ${this.info.lines}`, - ], this.options, this.__infoField()) - ); - if(err !== undefined) return err; + if(end === true){ + // Ensure we are not ending in a quoting state + if(this.state.quoting === true){ + const err = this.__error( + new CsvError$1('CSV_QUOTE_NOT_CLOSED', [ + 'Quote Not Closed:', + `the parsing is finished with an opening quote at line ${this.info.lines}`, + ], this.options, this.__infoField()) + ); + if(err !== undefined) return err; + }else { + // Skip last line if it has no characters + if(this.state.wasQuoting === true || this.state.record.length !== 0 || this.state.field.length !== 0){ + this.info.bytes = this.state.bufBytesStart + pos; + const errField = this.__onField(); + if(errField !== undefined) return errField; + const errRecord = this.__onRecord(push); + if(errRecord !== undefined) return errRecord; + }else if(this.state.wasRowDelimiter === true){ + this.info.empty_lines++; + }else if(this.state.commenting === true){ + this.info.comment_lines++; + } + } }else { - // Skip last line if it has no characters - if(this.state.wasQuoting === true || this.state.record.length !== 0 || this.state.field.length !== 0){ - this.info.bytes = this.state.bufBytesStart + pos; - const errField = this.__onField(); - if(errField !== undefined) return errField; - const errRecord = this.__onRecord(); - if(errRecord !== undefined) return errRecord; - }else if(this.state.wasRowDelimiter === true){ - this.info.empty_lines++; - }else if(this.state.commenting === true){ - this.info.comment_lines++; + this.state.bufBytesStart += pos; + this.state.previousBuf = buf.slice(pos); + } + if(this.state.wasRowDelimiter === true){ + this.info.lines++; + this.state.wasRowDelimiter = false; + } + }, + __onRecord: function(push){ + const {columns, group_columns_by_name, encoding, info, from, relax_column_count, relax_column_count_less, relax_column_count_more, raw, skip_records_with_empty_values} = this.options; + const {enabled, record} = this.state; + if(enabled === false){ + return this.__resetRecord(); + } + // Convert the first line into column names + const recordLength = record.length; + if(columns === true){ + if(skip_records_with_empty_values === true && isRecordEmpty(record)){ + this.__resetRecord(); + return; + } + return this.__firstLineToColumns(record); + } + if(columns === false && this.info.records === 0){ + this.state.expectedRecordLength = recordLength; + } + if(recordLength !== this.state.expectedRecordLength){ + const err = columns === false ? + new CsvError$1('CSV_RECORD_INCONSISTENT_FIELDS_LENGTH', [ + 'Invalid Record Length:', + `expect ${this.state.expectedRecordLength},`, + `got ${recordLength} on line ${this.info.lines}`, + ], this.options, this.__infoField(), { + record: record, + }) + : + new CsvError$1('CSV_RECORD_INCONSISTENT_COLUMNS', [ + 'Invalid Record Length:', + `columns length is ${columns.length},`, // rename columns + `got ${recordLength} on line ${this.info.lines}`, + ], this.options, this.__infoField(), { + record: record, + }); + if(relax_column_count === true || + (relax_column_count_less === true && recordLength < this.state.expectedRecordLength) || + (relax_column_count_more === true && recordLength > this.state.expectedRecordLength)){ + this.info.invalid_field_length++; + this.state.error = err; + // Error is undefined with skip_records_with_error + }else { + const finalErr = this.__error(err); + if(finalErr) return finalErr; } } - }else { - this.state.bufBytesStart += pos; - this.state.previousBuf = buf.slice(pos); - } - if(this.state.wasRowDelimiter === true){ - this.info.lines++; - this.state.wasRowDelimiter = false; - } - } - __onRecord(){ - const {columns, group_columns_by_name, encoding, info, from, relax_column_count, relax_column_count_less, relax_column_count_more, raw, skip_records_with_empty_values} = this.options; - const {enabled, record} = this.state; - if(enabled === false){ - return this.__resetRecord(); - } - // Convert the first line into column names - const recordLength = record.length; - if(columns === true){ if(skip_records_with_empty_values === true && isRecordEmpty(record)){ this.__resetRecord(); return; } - return this.__firstLineToColumns(record); - } - if(columns === false && this.info.records === 0){ - this.state.expectedRecordLength = recordLength; - } - if(recordLength !== this.state.expectedRecordLength){ - const err = columns === false ? - new CsvError$1('CSV_RECORD_INCONSISTENT_FIELDS_LENGTH', [ - 'Invalid Record Length:', - `expect ${this.state.expectedRecordLength},`, - `got ${recordLength} on line ${this.info.lines}`, - ], this.options, this.__infoField(), { - record: record, - }) - : - new CsvError$1('CSV_RECORD_INCONSISTENT_COLUMNS', [ - 'Invalid Record Length:', - `columns length is ${columns.length},`, // rename columns - `got ${recordLength} on line ${this.info.lines}`, - ], this.options, this.__infoField(), { - record: record, - }); - if(relax_column_count === true || - (relax_column_count_less === true && recordLength < this.state.expectedRecordLength) || - (relax_column_count_more === true && recordLength > this.state.expectedRecordLength)){ - this.info.invalid_field_length++; - this.state.error = err; - // Error is undefined with skip_records_with_error - }else { - const finalErr = this.__error(err); - if(finalErr) return finalErr; + if(this.state.recordHasError === true){ + this.__resetRecord(); + this.state.recordHasError = false; + return; } - } - if(skip_records_with_empty_values === true && isRecordEmpty(record)){ - this.__resetRecord(); - return; - } - if(this.state.recordHasError === true){ - this.__resetRecord(); - this.state.recordHasError = false; - return; - } - this.info.records++; - if(from === 1 || this.info.records >= from){ - const {objname} = this.options; - // With columns, records are object - if(columns !== false){ - const obj = {}; - // Transform record array to an object - for(let i = 0, l = record.length; i < l; i++){ - if(columns[i] === undefined || columns[i].disabled) continue; - // Turn duplicate columns into an array - if (group_columns_by_name === true && obj[columns[i].name] !== undefined) { - if (Array.isArray(obj[columns[i].name])) { - obj[columns[i].name] = obj[columns[i].name].concat(record[i]); + this.info.records++; + if(from === 1 || this.info.records >= from){ + const {objname} = this.options; + // With columns, records are object + if(columns !== false){ + const obj = {}; + // Transform record array to an object + for(let i = 0, l = record.length; i < l; i++){ + if(columns[i] === undefined || columns[i].disabled) continue; + // Turn duplicate columns into an array + if (group_columns_by_name === true && obj[columns[i].name] !== undefined) { + if (Array.isArray(obj[columns[i].name])) { + obj[columns[i].name] = obj[columns[i].name].concat(record[i]); + } else { + obj[columns[i].name] = [obj[columns[i].name], record[i]]; + } } else { - obj[columns[i].name] = [obj[columns[i].name], record[i]]; + obj[columns[i].name] = record[i]; } - } else { - obj[columns[i].name] = record[i]; } - } - // Without objname (default) - if(raw === true || info === true){ - const extRecord = Object.assign( - {record: obj}, - (raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}), - (info === true ? {info: this.__infoRecord()}: {}) - ); - const err = this.__push( - objname === undefined ? extRecord : [obj[objname], extRecord] - ); - if(err){ - return err; - } - }else { - const err = this.__push( - objname === undefined ? obj : [obj[objname], obj] - ); - if(err){ - return err; - } - } - // Without columns, records are array - }else { - if(raw === true || info === true){ - const extRecord = Object.assign( - {record: record}, - raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}, - info === true ? {info: this.__infoRecord()}: {} - ); - const err = this.__push( - objname === undefined ? extRecord : [record[objname], extRecord] - ); - if(err){ - return err; + // Without objname (default) + if(raw === true || info === true){ + const extRecord = Object.assign( + {record: obj}, + (raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}), + (info === true ? {info: this.__infoRecord()}: {}) + ); + const err = this.__push( + objname === undefined ? extRecord : [obj[objname], extRecord] + , push); + if(err){ + return err; + } + }else { + const err = this.__push( + objname === undefined ? obj : [obj[objname], obj] + , push); + if(err){ + return err; + } } + // Without columns, records are array }else { - const err = this.__push( - objname === undefined ? record : [record[objname], record] - ); - if(err){ - return err; + if(raw === true || info === true){ + const extRecord = Object.assign( + {record: record}, + raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}, + info === true ? {info: this.__infoRecord()}: {} + ); + const err = this.__push( + objname === undefined ? extRecord : [record[objname], extRecord] + , push); + if(err){ + return err; + } + }else { + const err = this.__push( + objname === undefined ? record : [record[objname], record] + , push); + if(err){ + return err; + } } } } - } - this.__resetRecord(); - } - __firstLineToColumns(record){ - const {firstLineToHeaders} = this.state; - try{ - const headers = firstLineToHeaders === undefined ? record : firstLineToHeaders.call(null, record); - if(!Array.isArray(headers)){ - return this.__error( - new CsvError$1('CSV_INVALID_COLUMN_MAPPING', [ - 'Invalid Column Mapping:', - 'expect an array from column function,', - `got ${JSON.stringify(headers)}` - ], this.options, this.__infoField(), { - headers: headers, - }) - ); - } - const normalizedHeaders = normalizeColumnsArray(headers); - this.state.expectedRecordLength = normalizedHeaders.length; - this.options.columns = normalizedHeaders; this.__resetRecord(); - return; - }catch(err){ - return err; - } - } - __resetRecord(){ - if(this.options.raw === true){ - this.state.rawBuffer.reset(); - } - this.state.error = undefined; - this.state.record = []; - this.state.record_length = 0; - } - __onField(){ - const {cast, encoding, rtrim, max_record_size} = this.options; - const {enabled, wasQuoting} = this.state; - // Short circuit for the from_line options - if(enabled === false){ - return this.__resetField(); - } - let field = this.state.field.toString(encoding); - if(rtrim === true && wasQuoting === false){ - field = field.trimRight(); - } - if(cast === true){ - const [err, f] = this.__cast(field); - if(err !== undefined) return err; - field = f; - } - this.state.record.push(field); - // Increment record length if record size must not exceed a limit - if(max_record_size !== 0 && typeof field === 'string'){ - this.state.record_length += field.length; - } - this.__resetField(); - } - __resetField(){ - this.state.field.reset(); - this.state.wasQuoting = false; - } - __push(record){ - const {on_record} = this.options; - if(on_record !== undefined){ - const info = this.__infoRecord(); + }, + __firstLineToColumns: function(record){ + const {firstLineToHeaders} = this.state; try{ - record = on_record.call(null, record, info); + const headers = firstLineToHeaders === undefined ? record : firstLineToHeaders.call(null, record); + if(!Array.isArray(headers)){ + return this.__error( + new CsvError$1('CSV_INVALID_COLUMN_MAPPING', [ + 'Invalid Column Mapping:', + 'expect an array from column function,', + `got ${JSON.stringify(headers)}` + ], this.options, this.__infoField(), { + headers: headers, + }) + ); + } + const normalizedHeaders = normalize_columns_array(headers); + this.state.expectedRecordLength = normalizedHeaders.length; + this.options.columns = normalizedHeaders; + this.__resetRecord(); + return; }catch(err){ return err; } - if(record === undefined || record === null){ return; } - } - this.push(record); - } - // Return a tuple with the error and the casted value - __cast(field){ - const {columns, relax_column_count} = this.options; - const isColumns = Array.isArray(columns); - // Dont loose time calling cast - // because the final record is an object - // and this field can't be associated to a key present in columns - if(isColumns === true && relax_column_count && this.options.columns.length <= this.state.record.length){ - return [undefined, undefined]; - } - if(this.state.castField !== null){ - try{ + }, + __resetRecord: function(){ + if(this.options.raw === true){ + this.state.rawBuffer.reset(); + } + this.state.error = undefined; + this.state.record = []; + this.state.record_length = 0; + }, + __onField: function(){ + const {cast, encoding, rtrim, max_record_size} = this.options; + const {enabled, wasQuoting} = this.state; + // Short circuit for the from_line options + if(enabled === false){ + return this.__resetField(); + } + let field = this.state.field.toString(encoding); + if(rtrim === true && wasQuoting === false){ + field = field.trimRight(); + } + if(cast === true){ + const [err, f] = this.__cast(field); + if(err !== undefined) return err; + field = f; + } + this.state.record.push(field); + // Increment record length if record size must not exceed a limit + if(max_record_size !== 0 && typeof field === 'string'){ + this.state.record_length += field.length; + } + this.__resetField(); + }, + __resetField: function(){ + this.state.field.reset(); + this.state.wasQuoting = false; + }, + __push: function(record, push){ + const {on_record} = this.options; + if(on_record !== undefined){ + const info = this.__infoRecord(); + try{ + record = on_record.call(null, record, info); + }catch(err){ + return err; + } + if(record === undefined || record === null){ return; } + } + push(record); + }, + // Return a tuple with the error and the casted value + __cast: function(field){ + const {columns, relax_column_count} = this.options; + const isColumns = Array.isArray(columns); + // Dont loose time calling cast + // because the final record is an object + // and this field can't be associated to a key present in columns + if(isColumns === true && relax_column_count && this.options.columns.length <= this.state.record.length){ + return [undefined, undefined]; + } + if(this.state.castField !== null){ + try{ + const info = this.__infoField(); + return [undefined, this.state.castField.call(null, field, info)]; + }catch(err){ + return [err]; + } + } + if(this.__isFloat(field)){ + return [undefined, parseFloat(field)]; + }else if(this.options.cast_date !== false){ const info = this.__infoField(); - return [undefined, this.state.castField.call(null, field, info)]; - }catch(err){ - return [err]; + return [undefined, this.options.cast_date.call(null, field, info)]; } - } - if(this.__isFloat(field)){ - return [undefined, parseFloat(field)]; - }else if(this.options.cast_date !== false){ - const info = this.__infoField(); - return [undefined, this.options.cast_date.call(null, field, info)]; - } - return [undefined, field]; - } - // Helper to test if a character is a space or a line delimiter - __isCharTrimable(chr){ - return chr === space || chr === tab || chr === cr || chr === nl || chr === np; - } - // Keep it in case we implement the `cast_int` option - // __isInt(value){ - // // return Number.isInteger(parseInt(value)) - // // return !isNaN( parseInt( obj ) ); - // return /^(\-|\+)?[1-9][0-9]*$/.test(value) - // } - __isFloat(value){ - return (value - parseFloat(value) + 1) >= 0; // Borrowed from jquery - } - __compareBytes(sourceBuf, targetBuf, targetPos, firstByte){ - if(sourceBuf[0] !== firstByte) return 0; - const sourceLength = sourceBuf.length; - for(let i = 1; i < sourceLength; i++){ - if(sourceBuf[i] !== targetBuf[targetPos+i]) return 0; - } - return sourceLength; - } - __needMoreData(i, bufLen, end){ - if(end) return false; - const {quote} = this.options; - const {quoting, needMoreDataSize, recordDelimiterMaxLength} = this.state; - const numOfCharLeft = bufLen - i - 1; - const requiredLength = Math.max( - needMoreDataSize, - // Skip if the remaining buffer smaller than record delimiter - recordDelimiterMaxLength, - // Skip if the remaining buffer can be record delimiter following the closing quote - // 1 is for quote.length - quoting ? (quote.length + recordDelimiterMaxLength) : 0, - ); - return numOfCharLeft < requiredLength; - } - __isDelimiter(buf, pos, chr){ - const {delimiter, ignore_last_delimiters} = this.options; - if(ignore_last_delimiters === true && this.state.record.length === this.options.columns.length - 1){ - return 0; - }else if(ignore_last_delimiters !== false && typeof ignore_last_delimiters === 'number' && this.state.record.length === ignore_last_delimiters - 1){ - return 0; - } - loop1: for(let i = 0; i < delimiter.length; i++){ - const del = delimiter[i]; - if(del[0] === chr){ - for(let j = 1; j < del.length; j++){ - if(del[j] !== buf[pos+j]) continue loop1; + return [undefined, field]; + }, + // Helper to test if a character is a space or a line delimiter + __isCharTrimable: function(chr){ + return chr === space || chr === tab || chr === cr || chr === nl || chr === np; + }, + // Keep it in case we implement the `cast_int` option + // __isInt(value){ + // // return Number.isInteger(parseInt(value)) + // // return !isNaN( parseInt( obj ) ); + // return /^(\-|\+)?[1-9][0-9]*$/.test(value) + // } + __isFloat: function(value){ + return (value - parseFloat(value) + 1) >= 0; // Borrowed from jquery + }, + __compareBytes: function(sourceBuf, targetBuf, targetPos, firstByte){ + if(sourceBuf[0] !== firstByte) return 0; + const sourceLength = sourceBuf.length; + for(let i = 1; i < sourceLength; i++){ + if(sourceBuf[i] !== targetBuf[targetPos+i]) return 0; + } + return sourceLength; + }, + __isDelimiter: function(buf, pos, chr){ + const {delimiter, ignore_last_delimiters} = this.options; + if(ignore_last_delimiters === true && this.state.record.length === this.options.columns.length - 1){ + return 0; + }else if(ignore_last_delimiters !== false && typeof ignore_last_delimiters === 'number' && this.state.record.length === ignore_last_delimiters - 1){ + return 0; + } + loop1: for(let i = 0; i < delimiter.length; i++){ + const del = delimiter[i]; + if(del[0] === chr){ + for(let j = 1; j < del.length; j++){ + if(del[j] !== buf[pos+j]) continue loop1; + } + return del.length; } - return del.length; } - } - return 0; - } - __isRecordDelimiter(chr, buf, pos){ - const {record_delimiter} = this.options; - const recordDelimiterLength = record_delimiter.length; - loop1: for(let i = 0; i < recordDelimiterLength; i++){ - const rd = record_delimiter[i]; - const rdLength = rd.length; - if(rd[0] !== chr){ - continue; + return 0; + }, + __isRecordDelimiter: function(chr, buf, pos){ + const {record_delimiter} = this.options; + const recordDelimiterLength = record_delimiter.length; + loop1: for(let i = 0; i < recordDelimiterLength; i++){ + const rd = record_delimiter[i]; + const rdLength = rd.length; + if(rd[0] !== chr){ + continue; + } + for(let j = 1; j < rdLength; j++){ + if(rd[j] !== buf[pos+j]){ + continue loop1; + } + } + return rd.length; } - for(let j = 1; j < rdLength; j++){ - if(rd[j] !== buf[pos+j]){ - continue loop1; + return 0; + }, + __isEscape: function(buf, pos, chr){ + const {escape} = this.options; + if(escape === null) return false; + const l = escape.length; + if(escape[0] === chr){ + for(let i = 0; i < l; i++){ + if(escape[i] !== buf[pos+i]){ + return false; + } } + return true; } - return rd.length; - } - return 0; - } - __isEscape(buf, pos, chr){ - const {escape} = this.options; - if(escape === null) return false; - const l = escape.length; - if(escape[0] === chr){ + return false; + }, + __isQuote: function(buf, pos){ + const {quote} = this.options; + if(quote === null) return false; + const l = quote.length; for(let i = 0; i < l; i++){ - if(escape[i] !== buf[pos+i]){ + if(quote[i] !== buf[pos+i]){ return false; } } return true; - } - return false; - } - __isQuote(buf, pos){ - const {quote} = this.options; - if(quote === null) return false; - const l = quote.length; - for(let i = 0; i < l; i++){ - if(quote[i] !== buf[pos+i]){ - return false; - } - } - return true; - } - __autoDiscoverRecordDelimiter(buf, pos){ - const {encoding} = this.options; - const chr = buf[pos]; - if(chr === cr){ - if(buf[pos+1] === nl){ - this.options.record_delimiter.push(Buffer.from('\r\n', encoding)); - this.state.recordDelimiterMaxLength = 2; - return 2; - }else { - this.options.record_delimiter.push(Buffer.from('\r', encoding)); + }, + __autoDiscoverRecordDelimiter: function(buf, pos){ + const {encoding} = this.options; + const chr = buf[pos]; + if(chr === cr){ + if(buf[pos+1] === nl){ + this.options.record_delimiter.push(Buffer.from('\r\n', encoding)); + this.state.recordDelimiterMaxLength = 2; + return 2; + }else { + this.options.record_delimiter.push(Buffer.from('\r', encoding)); + this.state.recordDelimiterMaxLength = 1; + return 1; + } + }else if(chr === nl){ + this.options.record_delimiter.push(Buffer.from('\n', encoding)); this.state.recordDelimiterMaxLength = 1; return 1; } - }else if(chr === nl){ - this.options.record_delimiter.push(Buffer.from('\n', encoding)); - this.state.recordDelimiterMaxLength = 1; - return 1; - } - return 0; - } - __error(msg){ - const {encoding, raw, skip_records_with_error} = this.options; - const err = typeof msg === 'string' ? new Error(msg) : msg; - if(skip_records_with_error){ - this.state.recordHasError = true; - this.emit('skip', err, raw ? this.state.rawBuffer.toString(encoding) : undefined); - return undefined; - }else { - return err; + return 0; + }, + __error: function(msg){ + const {encoding, raw, skip_records_with_error} = this.options; + const err = typeof msg === 'string' ? new Error(msg) : msg; + if(skip_records_with_error){ + this.state.recordHasError = true; + if(this.options.on_skip !== undefined){ + this.options.on_skip(err, raw ? this.state.rawBuffer.toString(encoding) : undefined); + } + // this.emit('skip', err, raw ? this.state.rawBuffer.toString(encoding) : undefined); + return undefined; + }else { + return err; + } + }, + __infoDataSet: function(){ + return { + ...this.info, + columns: this.options.columns + }; + }, + __infoRecord: function(){ + const {columns, raw, encoding} = this.options; + return { + ...this.__infoDataSet(), + error: this.state.error, + header: columns === true, + index: this.state.record.length, + raw: raw ? this.state.rawBuffer.toString(encoding) : undefined + }; + }, + __infoField: function(){ + const {columns} = this.options; + const isColumns = Array.isArray(columns); + return { + ...this.__infoRecord(), + column: isColumns === true ? + (columns.length > this.state.record.length ? + columns[this.state.record.length].name : + null + ) : + this.state.record.length, + quoting: this.state.wasQuoting, + }; } - } - __infoDataSet(){ - return { - ...this.info, - columns: this.options.columns + }; + }; + + class Parser extends Transform { + constructor(opts = {}){ + super({...{readableObjectMode: true}, ...opts, encoding: null}); + this.api = transform$1(opts); + this.api.options.on_skip = (err, chunk) => { + this.emit('skip', err, chunk); }; + // Backward compatibility + this.state = this.api.state; + this.options = this.api.options; + this.info = this.api.info; } - __infoRecord(){ - const {columns, raw, encoding} = this.options; - return { - ...this.__infoDataSet(), - error: this.state.error, - header: columns === true, - index: this.state.record.length, - raw: raw ? this.state.rawBuffer.toString(encoding) : undefined - }; + // Implementation of `Transform._transform` + _transform(buf, encoding, callback){ + if(this.state.stop === true){ + return; + } + const err = this.api.parse(buf, false, (record) => { + this.push.call(this, record); + }, () => { + this.push.call(this, null); + }); + if(err !== undefined){ + this.state.stop = true; + } + callback(err); } - __infoField(){ - const {columns} = this.options; - const isColumns = Array.isArray(columns); - return { - ...this.__infoRecord(), - column: isColumns === true ? - (columns.length > this.state.record.length ? - columns[this.state.record.length].name : - null - ) : - this.state.record.length, - quoting: this.state.wasQuoting, - }; + // Implementation of `Transform._flush` + _flush(callback){ + if(this.state.stop === true){ + return; + } + const err = this.api.parse(undefined, true, (record) => { + this.push.call(this, record); + }, () => { + this.push.call(this, null); + }); + callback(err); } } @@ -6534,7 +6598,7 @@ var csv = (function (exports) { const type = typeof argument; if(data === undefined && (typeof argument === 'string' || isBuffer$1(argument))){ data = argument; - }else if(options === undefined && isObject$1(argument)){ + }else if(options === undefined && is_object$1(argument)){ options = argument; }else if(callback === undefined && type === 'function'){ callback = argument; @@ -6559,10 +6623,10 @@ var csv = (function (exports) { } }); parser.on('error', function(err){ - callback(err, undefined, parser.__infoDataSet()); + callback(err, undefined, parser.api.__infoDataSet()); }); parser.on('end', function(){ - callback(undefined, records, parser.__infoDataSet()); + callback(undefined, records, parser.api.__infoDataSet()); }); } if(data !== undefined){ @@ -6725,8 +6789,6 @@ var csv = (function (exports) { return transformer; }; - const bom_utf8 = Buffer.from([239, 187, 191]); - class CsvError extends Error { constructor(code, message, ...contexts) { if(Array.isArray(message)) message = message.join(' '); @@ -6744,16 +6806,10 @@ var csv = (function (exports) { } } - const isObject = function(obj){ + const is_object = function(obj){ return typeof obj === 'object' && obj !== null && ! Array.isArray(obj); }; - const underscore = function(str){ - return str.replace(/([A-Z])/g, function(_, match){ - return '_' + match.toLowerCase(); - }); - }; - // Lodash implementation of `get` const charCodeOfDot = '.'.charCodeAt(0); @@ -6831,441 +6887,472 @@ var csv = (function (exports) { return (index && index === length) ? object : undefined; }; - class Stringifier extends Transform { - constructor(opts = {}){ - super({...{writableObjectMode: true}, ...opts}); - const options = {}; - let err; - // Merge with user options - for(const opt in opts){ - options[underscore(opt)] = opts[opt]; - } - if((err = this.normalize(options)) !== undefined) throw err; - switch(options.record_delimiter){ - case 'auto': - options.record_delimiter = null; - break; - case 'unix': - options.record_delimiter = "\n"; - break; - case 'mac': - options.record_delimiter = "\r"; - break; - case 'windows': - options.record_delimiter = "\r\n"; - break; - case 'ascii': - options.record_delimiter = "\u001e"; - break; - case 'unicode': - options.record_delimiter = "\u2028"; - break; - } - // Expose options - this.options = options; - // Internal state - this.state = { - stop: false - }; - // Information - this.info = { - records: 0 - }; + const normalize_columns = function(columns){ + if(columns === undefined || columns === null){ + return [undefined, undefined]; } - normalize(options){ - // Normalize option `bom` - if(options.bom === undefined || options.bom === null || options.bom === false){ - options.bom = false; - }else if(options.bom !== true){ - return new CsvError('CSV_OPTION_BOOLEAN_INVALID_TYPE', [ - 'option `bom` is optional and must be a boolean value,', - `got ${JSON.stringify(options.bom)}` - ]); - } - // Normalize option `delimiter` - if(options.delimiter === undefined || options.delimiter === null){ - options.delimiter = ','; - }else if(isBuffer$1(options.delimiter)){ - options.delimiter = options.delimiter.toString(); - }else if(typeof options.delimiter !== 'string'){ - return new CsvError('CSV_OPTION_DELIMITER_INVALID_TYPE', [ - 'option `delimiter` must be a buffer or a string,', - `got ${JSON.stringify(options.delimiter)}` - ]); - } - // Normalize option `quote` - if(options.quote === undefined || options.quote === null){ - options.quote = '"'; - }else if(options.quote === true){ - options.quote = '"'; - }else if(options.quote === false){ - options.quote = ''; - }else if (isBuffer$1(options.quote)){ - options.quote = options.quote.toString(); - }else if(typeof options.quote !== 'string'){ - return new CsvError('CSV_OPTION_QUOTE_INVALID_TYPE', [ - 'option `quote` must be a boolean, a buffer or a string,', - `got ${JSON.stringify(options.quote)}` - ]); + if(typeof columns !== 'object'){ + return [Error('Invalid option "columns": expect an array or an object')]; + } + if(!Array.isArray(columns)){ + const newcolumns = []; + for(const k in columns){ + newcolumns.push({ + key: k, + header: columns[k] + }); } - // Normalize option `quoted` - if(options.quoted === undefined || options.quoted === null){ - options.quoted = false; - } - // Normalize option `quoted_empty` - if(options.quoted_empty === undefined || options.quoted_empty === null){ - options.quoted_empty = undefined; - } - // Normalize option `quoted_match` - if(options.quoted_match === undefined || options.quoted_match === null || options.quoted_match === false){ - options.quoted_match = null; - }else if(!Array.isArray(options.quoted_match)){ - options.quoted_match = [options.quoted_match]; - } - if(options.quoted_match){ - for(const quoted_match of options.quoted_match){ - const isString = typeof quoted_match === 'string'; - const isRegExp = quoted_match instanceof RegExp; - if(!isString && !isRegExp){ - return Error(`Invalid Option: quoted_match must be a string or a regex, got ${JSON.stringify(quoted_match)}`); + columns = newcolumns; + }else { + const newcolumns = []; + for(const column of columns){ + if(typeof column === 'string'){ + newcolumns.push({ + key: column, + header: column + }); + }else if(typeof column === 'object' && column !== null && !Array.isArray(column)){ + if(!column.key){ + return [Error('Invalid column definition: property "key" is required')]; } + if(column.header === undefined){ + column.header = column.key; + } + newcolumns.push(column); + }else { + return [Error('Invalid column definition: expect a string or an object')]; } } - // Normalize option `quoted_string` - if(options.quoted_string === undefined || options.quoted_string === null){ - options.quoted_string = false; - } - // Normalize option `eof` - if(options.eof === undefined || options.eof === null){ - options.eof = true; - } - // Normalize option `escape` - if(options.escape === undefined || options.escape === null){ - options.escape = '"'; - }else if(isBuffer$1(options.escape)){ - options.escape = options.escape.toString(); - }else if(typeof options.escape !== 'string'){ - return Error(`Invalid Option: escape must be a buffer or a string, got ${JSON.stringify(options.escape)}`); - } - if (options.escape.length > 1){ - return Error(`Invalid Option: escape must be one character, got ${options.escape.length} characters`); - } - // Normalize option `header` - if(options.header === undefined || options.header === null){ - options.header = false; - } - // Normalize option `columns` - const [err, columns] = this.normalize_columns(options.columns); - if(err) return err; - options.columns = columns; - // Normalize option `quoted` - if(options.quoted === undefined || options.quoted === null){ - options.quoted = false; - } - // Normalize option `cast` - if(options.cast === undefined || options.cast === null){ - options.cast = {}; - } - // Normalize option cast.bigint - if(options.cast.bigint === undefined || options.cast.bigint === null){ - // Cast boolean to string by default - options.cast.bigint = value => '' + value; - } - // Normalize option cast.boolean - if(options.cast.boolean === undefined || options.cast.boolean === null){ - // Cast boolean to string by default - options.cast.boolean = value => value ? '1' : ''; - } - // Normalize option cast.date - if(options.cast.date === undefined || options.cast.date === null){ - // Cast date to timestamp string by default - options.cast.date = value => '' + value.getTime(); - } - // Normalize option cast.number - if(options.cast.number === undefined || options.cast.number === null){ - // Cast number to string using native casting by default - options.cast.number = value => '' + value; - } - // Normalize option cast.object - if(options.cast.object === undefined || options.cast.object === null){ - // Stringify object as JSON by default - options.cast.object = value => JSON.stringify(value); - } - // Normalize option cast.string - if(options.cast.string === undefined || options.cast.string === null){ - // Leave string untouched - options.cast.string = function(value){return value;}; - } - // Normalize option `record_delimiter` - if(options.record_delimiter === undefined || options.record_delimiter === null){ - options.record_delimiter = '\n'; - }else if(isBuffer$1(options.record_delimiter)){ - options.record_delimiter = options.record_delimiter.toString(); - }else if(typeof options.record_delimiter !== 'string'){ - return Error(`Invalid Option: record_delimiter must be a buffer or a string, got ${JSON.stringify(options.record_delimiter)}`); - } + columns = newcolumns; } - _transform(chunk, encoding, callback){ - if(this.state.stop === true){ - return; - } - const err = this.__transform(chunk); - if(err !== undefined){ - this.state.stop = true; + return [undefined, columns]; + }; + + const underscore = function(str){ + return str.replace(/([A-Z])/g, function(_, match){ + return '_' + match.toLowerCase(); + }); + }; + + const normalize_options = function(opts) { + const options = {}; + // Merge with user options + for(const opt in opts){ + options[underscore(opt)] = opts[opt]; + } + // Normalize option `bom` + if(options.bom === undefined || options.bom === null || options.bom === false){ + options.bom = false; + }else if(options.bom !== true){ + return [new CsvError('CSV_OPTION_BOOLEAN_INVALID_TYPE', [ + 'option `bom` is optional and must be a boolean value,', + `got ${JSON.stringify(options.bom)}` + ])]; + } + // Normalize option `delimiter` + if(options.delimiter === undefined || options.delimiter === null){ + options.delimiter = ','; + }else if(isBuffer$1(options.delimiter)){ + options.delimiter = options.delimiter.toString(); + }else if(typeof options.delimiter !== 'string'){ + return [new CsvError('CSV_OPTION_DELIMITER_INVALID_TYPE', [ + 'option `delimiter` must be a buffer or a string,', + `got ${JSON.stringify(options.delimiter)}` + ])]; + } + // Normalize option `quote` + if(options.quote === undefined || options.quote === null){ + options.quote = '"'; + }else if(options.quote === true){ + options.quote = '"'; + }else if(options.quote === false){ + options.quote = ''; + }else if (isBuffer$1(options.quote)){ + options.quote = options.quote.toString(); + }else if(typeof options.quote !== 'string'){ + return [new CsvError('CSV_OPTION_QUOTE_INVALID_TYPE', [ + 'option `quote` must be a boolean, a buffer or a string,', + `got ${JSON.stringify(options.quote)}` + ])]; + } + // Normalize option `quoted` + if(options.quoted === undefined || options.quoted === null){ + options.quoted = false; + } + // Normalize option `quoted_empty` + if(options.quoted_empty === undefined || options.quoted_empty === null){ + options.quoted_empty = undefined; + } + // Normalize option `quoted_match` + if(options.quoted_match === undefined || options.quoted_match === null || options.quoted_match === false){ + options.quoted_match = null; + }else if(!Array.isArray(options.quoted_match)){ + options.quoted_match = [options.quoted_match]; + } + if(options.quoted_match){ + for(const quoted_match of options.quoted_match){ + const isString = typeof quoted_match === 'string'; + const isRegExp = quoted_match instanceof RegExp; + if(!isString && !isRegExp){ + return [Error(`Invalid Option: quoted_match must be a string or a regex, got ${JSON.stringify(quoted_match)}`)]; + } } - callback(err); } - _flush(callback){ - if(this.state.stop === true){ - // Note, Node.js 12 call flush even after an error, we must prevent - // `callback` from being called in flush without any error. - return; - } - if(this.info.records === 0){ - this.bom(); - const err = this.headers(); - if(err) callback(err); - } - callback(); + // Normalize option `quoted_string` + if(options.quoted_string === undefined || options.quoted_string === null){ + options.quoted_string = false; + } + // Normalize option `eof` + if(options.eof === undefined || options.eof === null){ + options.eof = true; + } + // Normalize option `escape` + if(options.escape === undefined || options.escape === null){ + options.escape = '"'; + }else if(isBuffer$1(options.escape)){ + options.escape = options.escape.toString(); + }else if(typeof options.escape !== 'string'){ + return [Error(`Invalid Option: escape must be a buffer or a string, got ${JSON.stringify(options.escape)}`)]; + } + if (options.escape.length > 1){ + return [Error(`Invalid Option: escape must be one character, got ${options.escape.length} characters`)]; + } + // Normalize option `header` + if(options.header === undefined || options.header === null){ + options.header = false; + } + // Normalize option `columns` + const [errColumns, columns] = normalize_columns(options.columns); + if(errColumns !== undefined) return [errColumns]; + options.columns = columns; + // Normalize option `quoted` + if(options.quoted === undefined || options.quoted === null){ + options.quoted = false; + } + // Normalize option `cast` + if(options.cast === undefined || options.cast === null){ + options.cast = {}; + } + // Normalize option cast.bigint + if(options.cast.bigint === undefined || options.cast.bigint === null){ + // Cast boolean to string by default + options.cast.bigint = value => '' + value; + } + // Normalize option cast.boolean + if(options.cast.boolean === undefined || options.cast.boolean === null){ + // Cast boolean to string by default + options.cast.boolean = value => value ? '1' : ''; + } + // Normalize option cast.date + if(options.cast.date === undefined || options.cast.date === null){ + // Cast date to timestamp string by default + options.cast.date = value => '' + value.getTime(); + } + // Normalize option cast.number + if(options.cast.number === undefined || options.cast.number === null){ + // Cast number to string using native casting by default + options.cast.number = value => '' + value; + } + // Normalize option cast.object + if(options.cast.object === undefined || options.cast.object === null){ + // Stringify object as JSON by default + options.cast.object = value => JSON.stringify(value); + } + // Normalize option cast.string + if(options.cast.string === undefined || options.cast.string === null){ + // Leave string untouched + options.cast.string = function(value){return value;}; + } + // Normalize option `on_record` + if(options.on_record !== undefined && typeof options.on_record !== 'function'){ + return [Error(`Invalid Option: "on_record" must be a function.`)]; + } + // Normalize option `record_delimiter` + if(options.record_delimiter === undefined || options.record_delimiter === null){ + options.record_delimiter = '\n'; + }else if(isBuffer$1(options.record_delimiter)){ + options.record_delimiter = options.record_delimiter.toString(); + }else if(typeof options.record_delimiter !== 'string'){ + return [Error(`Invalid Option: record_delimiter must be a buffer or a string, got ${JSON.stringify(options.record_delimiter)}`)]; + } + switch(options.record_delimiter){ + case 'auto': + options.record_delimiter = null; + break; + case 'unix': + options.record_delimiter = "\n"; + break; + case 'mac': + options.record_delimiter = "\r"; + break; + case 'windows': + options.record_delimiter = "\r\n"; + break; + case 'ascii': + options.record_delimiter = "\u001e"; + break; + case 'unicode': + options.record_delimiter = "\u2028"; + break; } - __transform(chunk){ - // Chunk validation - if(!Array.isArray(chunk) && typeof chunk !== 'object'){ - return Error(`Invalid Record: expect an array or an object, got ${JSON.stringify(chunk)}`); - } - // Detect columns from the first record - if(this.info.records === 0){ - if(Array.isArray(chunk)){ - if(this.options.header === true && this.options.columns === undefined){ - return Error('Undiscoverable Columns: header option requires column option or object records'); + return [undefined, options]; + }; + + const bom_utf8 = Buffer.from([239, 187, 191]); + + const stringifier = function(options, state, info){ + return { + options: options, + state: state, + info: info, + __transform: function(chunk, push){ + // Chunk validation + if(!Array.isArray(chunk) && typeof chunk !== 'object'){ + return Error(`Invalid Record: expect an array or an object, got ${JSON.stringify(chunk)}`); + } + // Detect columns from the first record + if(this.info.records === 0){ + if(Array.isArray(chunk)){ + if(this.options.header === true && this.options.columns === undefined){ + return Error('Undiscoverable Columns: header option requires column option or object records'); + } + }else if(this.options.columns === undefined){ + const [err, columns] = normalize_columns(Object.keys(chunk)); + if(err) return; + this.options.columns = columns; } - }else if(this.options.columns === undefined){ - const [err, columns] = this.normalize_columns(Object.keys(chunk)); - if(err) return; - this.options.columns = columns; } - } - // Emit the header - if(this.info.records === 0){ - this.bom(); - const err = this.headers(); - if(err) return err; - } - // Emit and stringify the record if an object or an array - try{ - this.emit('record', chunk, this.info.records); - }catch(err){ - return err; - } - // Convert the record into a string - let err, chunk_string; - if(this.options.eof){ - [err, chunk_string] = this.stringify(chunk); - if(err) return err; - if(chunk_string === undefined){ - return; - }else { - chunk_string = chunk_string + this.options.record_delimiter; + // Emit the header + if(this.info.records === 0){ + this.bom(push); + const err = this.headers(push); + if(err) return err; } - }else { - [err, chunk_string] = this.stringify(chunk); - if(err) return err; - if(chunk_string === undefined){ - return; - }else { - if(this.options.header || this.info.records){ - chunk_string = this.options.record_delimiter + chunk_string; + // Emit and stringify the record if an object or an array + try{ + // this.emit('record', chunk, this.info.records); + if(this.options.on_record){ + this.options.on_record(chunk, this.info.records); } + }catch(err){ + return err; } - } - // Emit the csv - this.info.records++; - this.push(chunk_string); - } - stringify(chunk, chunkIsHeader=false){ - if(typeof chunk !== 'object'){ - return [undefined, chunk]; - } - const {columns} = this.options; - const record = []; - // Record is an array - if(Array.isArray(chunk)){ - // We are getting an array but the user has specified output columns. In - // this case, we respect the columns indexes - if(columns){ - chunk.splice(columns.length); - } - // Cast record elements - for(let i=0; i= 0; - const containsQuote = (quote !== '') && value.indexOf(quote) >= 0; - const containsEscape = value.indexOf(escape) >= 0 && (escape !== quote); - const containsRecordDelimiter = value.indexOf(record_delimiter) >= 0; - const quotedString = quoted_string && typeof field === 'string'; - let quotedMatch = quoted_match && quoted_match.filter(quoted_match => { - if(typeof quoted_match === 'string'){ - return value.indexOf(quoted_match) !== -1; - }else { - return quoted_match.test(value); + } + let csvrecord = ''; + for(let i=0; i 0; - const shouldQuote = containsQuote === true || containsdelimiter || containsRecordDelimiter || quoted || quotedString || quotedMatch; - if(shouldQuote === true && containsEscape === true){ - const regexp = escape === '\\' - ? new RegExp(escape + escape, 'g') - : new RegExp(escape, 'g'); - value = value.replace(regexp, escape + escape); + options = {...this.options, ...options}; + [err, options] = normalize_options(options); + if(err !== undefined){ + return [err]; + } + }else if(value === undefined || value === null){ + options = this.options; + }else { + return [Error(`Invalid Casting Value: returned value must return a string, an object, null or undefined, got ${JSON.stringify(value)}`)]; } - if(containsQuote === true){ - const regexp = new RegExp(quote,'g'); - value = value.replace(regexp, escape + quote); + const {delimiter, escape, quote, quoted, quoted_empty, quoted_string, quoted_match, record_delimiter} = options; + if(value){ + if(typeof value !== 'string'){ + return [Error(`Formatter must return a string, null or undefined, got ${JSON.stringify(value)}`)]; + } + const containsdelimiter = delimiter.length && value.indexOf(delimiter) >= 0; + const containsQuote = (quote !== '') && value.indexOf(quote) >= 0; + const containsEscape = value.indexOf(escape) >= 0 && (escape !== quote); + const containsRecordDelimiter = value.indexOf(record_delimiter) >= 0; + const quotedString = quoted_string && typeof field === 'string'; + let quotedMatch = quoted_match && quoted_match.filter(quoted_match => { + if(typeof quoted_match === 'string'){ + return value.indexOf(quoted_match) !== -1; + }else { + return quoted_match.test(value); + } + }); + quotedMatch = quotedMatch && quotedMatch.length > 0; + const shouldQuote = containsQuote === true || containsdelimiter || containsRecordDelimiter || quoted || quotedString || quotedMatch; + if(shouldQuote === true && containsEscape === true){ + const regexp = escape === '\\' + ? new RegExp(escape + escape, 'g') + : new RegExp(escape, 'g'); + value = value.replace(regexp, escape + escape); + } + if(containsQuote === true){ + const regexp = new RegExp(quote,'g'); + value = value.replace(regexp, escape + quote); + } + if(shouldQuote === true){ + value = quote + value + quote; + } + csvrecord += value; + }else if(quoted_empty === true || (field === '' && quoted_string === true && quoted_empty !== false)){ + csvrecord += quote + quote; } - if(shouldQuote === true){ - value = quote + value + quote; + if(i !== record.length - 1){ + csvrecord += delimiter; } - csvrecord += value; - }else if(quoted_empty === true || (field === '' && quoted_string === true && quoted_empty !== false)){ - csvrecord += quote + quote; } - if(i !== record.length - 1){ - csvrecord += delimiter; + return [undefined, csvrecord]; + }, + bom: function(push){ + if(this.options.bom !== true){ + return; + } + push(bom_utf8); + }, + headers: function(push){ + if(this.options.header === false){ + return; + } + if(this.options.columns === undefined){ + return; + } + let err; + let headers = this.options.columns.map(column => column.header); + if(this.options.eof){ + [err, headers] = this.stringify(headers, true); + headers += this.options.record_delimiter; + }else { + [err, headers] = this.stringify(headers); + } + if(err) return err; + push(headers); + }, + __cast: function(value, context){ + const type = typeof value; + try{ + if(type === 'string'){ // Fine for 99% of the cases + return [undefined, this.options.cast.string(value, context)]; + }else if(type === 'bigint'){ + return [undefined, this.options.cast.bigint(value, context)]; + }else if(type === 'number'){ + return [undefined, this.options.cast.number(value, context)]; + }else if(type === 'boolean'){ + return [undefined, this.options.cast.boolean(value, context)]; + }else if(value instanceof Date){ + return [undefined, this.options.cast.date(value, context)]; + }else if(type === 'object' && value !== null){ + return [undefined, this.options.cast.object(value, context)]; + }else { + return [undefined, value, value]; + } + }catch(err){ + return [err]; } } - return [undefined, csvrecord]; - } - bom(){ - if(this.options.bom !== true){ - return; - } - this.push(bom_utf8); + }; + }; + + class Stringifier extends Transform { + constructor(opts = {}){ + super({...{writableObjectMode: true}, ...opts}); + const [err, options] = normalize_options(opts); + if(err !== undefined) throw err; + // Expose options + this.options = options; + // Internal state + this.state = { + stop: false + }; + // Information + this.info = { + records: 0 + }; + this.api = stringifier(this.options, this.state, this.info); + this.api.options.on_record = (...args) => { + this.emit('record', ...args); + }; } - headers(){ - if(this.options.header === false){ - return; - } - if(this.options.columns === undefined){ + _transform(chunk, encoding, callback){ + if(this.state.stop === true){ return; } - let err; - let headers = this.options.columns.map(column => column.header); - if(this.options.eof){ - [err, headers] = this.stringify(headers, true); - headers += this.options.record_delimiter; - }else { - [err, headers] = this.stringify(headers); - } - if(err) return err; - this.push(headers); - } - __cast(value, context){ - const type = typeof value; - try{ - if(type === 'string'){ // Fine for 99% of the cases - return [undefined, this.options.cast.string(value, context)]; - }else if(type === 'bigint'){ - return [undefined, this.options.cast.bigint(value, context)]; - }else if(type === 'number'){ - return [undefined, this.options.cast.number(value, context)]; - }else if(type === 'boolean'){ - return [undefined, this.options.cast.boolean(value, context)]; - }else if(value instanceof Date){ - return [undefined, this.options.cast.date(value, context)]; - }else if(type === 'object' && value !== null){ - return [undefined, this.options.cast.object(value, context)]; - }else { - return [undefined, value, value]; - } - }catch(err){ - return [err]; + const err = this.api.__transform(chunk, this.push.bind(this)); + if(err !== undefined){ + this.state.stop = true; } + callback(err); } - normalize_columns(columns){ - if(columns === undefined || columns === null){ - return []; - } - if(typeof columns !== 'object'){ - return [Error('Invalid option "columns": expect an array or an object')]; + _flush(callback){ + if(this.state.stop === true){ + // Note, Node.js 12 call flush even after an error, we must prevent + // `callback` from being called in flush without any error. + return; } - if(!Array.isArray(columns)){ - const newcolumns = []; - for(const k in columns){ - newcolumns.push({ - key: k, - header: columns[k] - }); - } - columns = newcolumns; - }else { - const newcolumns = []; - for(const column of columns){ - if(typeof column === 'string'){ - newcolumns.push({ - key: column, - header: column - }); - }else if(typeof column === 'object' && column !== undefined && !Array.isArray(column)){ - if(!column.key){ - return [Error('Invalid column definition: property "key" is required')]; - } - if(column.header === undefined){ - column.header = column.key; - } - newcolumns.push(column); - }else { - return [Error('Invalid column definition: expect a string or an object')]; - } - } - columns = newcolumns; + if(this.info.records === 0){ + this.api.bom(this.push.bind(this)); + const err = this.api.headers(this.push.bind(this)); + if(err) callback(err); } - return [undefined, columns]; + callback(); } } @@ -7276,7 +7363,7 @@ var csv = (function (exports) { const type = typeof argument; if(data === undefined && (Array.isArray(argument))){ data = argument; - }else if(options === undefined && isObject(argument)){ + }else if(options === undefined && is_object(argument)){ options = argument; }else if(callback === undefined && type === 'function'){ callback = argument; diff --git a/packages/csv/dist/iife/sync.js b/packages/csv/dist/iife/sync.js index 945d26f8f..c3696d179 100644 --- a/packages/csv/dist/iife/sync.js +++ b/packages/csv/dist/iife/sync.js @@ -115,7 +115,7 @@ var csv_sync = (function (exports) { return parts.join('') } - function read (buffer, offset, isLE, mLen, nBytes) { + function read$1 (buffer, offset, isLE, mLen, nBytes) { var e, m; var eLen = nBytes * 8 - mLen - 1; var eMax = (1 << eLen) - 1; @@ -1396,22 +1396,22 @@ var csv_sync = (function (exports) { Buffer.prototype.readFloatLE = function readFloatLE (offset, noAssert) { if (!noAssert) checkOffset(offset, 4, this.length); - return read(this, offset, true, 23, 4) + return read$1(this, offset, true, 23, 4) }; Buffer.prototype.readFloatBE = function readFloatBE (offset, noAssert) { if (!noAssert) checkOffset(offset, 4, this.length); - return read(this, offset, false, 23, 4) + return read$1(this, offset, false, 23, 4) }; Buffer.prototype.readDoubleLE = function readDoubleLE (offset, noAssert) { if (!noAssert) checkOffset(offset, 8, this.length); - return read(this, offset, true, 52, 8) + return read$1(this, offset, true, 52, 8) }; Buffer.prototype.readDoubleBE = function readDoubleBE (offset, noAssert) { if (!noAssert) checkOffset(offset, 8, this.length); - return read(this, offset, false, 52, 8) + return read$1(this, offset, false, 52, 8) }; function checkInt (buf, value, offset, ext, max, min) { @@ -2630,7 +2630,7 @@ var csv_sync = (function (exports) { } }); for (var x = args[i]; i < len; x = args[++i]) { - if (isNull(x) || !isObject$2(x)) { + if (isNull(x) || !isObject(x)) { str += ' ' + x; } else { str += ' ' + inspect(x); @@ -3046,19 +3046,19 @@ var csv_sync = (function (exports) { } function isRegExp(re) { - return isObject$2(re) && objectToString(re) === '[object RegExp]'; + return isObject(re) && objectToString(re) === '[object RegExp]'; } - function isObject$2(arg) { + function isObject(arg) { return typeof arg === 'object' && arg !== null; } function isDate(d) { - return isObject$2(d) && objectToString(d) === '[object Date]'; + return isObject(d) && objectToString(d) === '[object Date]'; } function isError(e) { - return isObject$2(e) && + return isObject(e) && (objectToString(e) === '[object Error]' || e instanceof Error); } @@ -3109,7 +3109,7 @@ var csv_sync = (function (exports) { function _extend(origin, add) { // Don't do anything if add isn't an object - if (!add || !isObject$2(add)) return origin; + if (!add || !isObject(add)) return origin; var keys = Object.keys(add); var i = keys.length; @@ -3131,7 +3131,7 @@ var csv_sync = (function (exports) { isFunction: isFunction, isError: isError, isDate: isDate, - isObject: isObject$2, + isObject: isObject, isRegExp: isRegExp, isUndefined: isUndefined, isSymbol: isSymbol$1, @@ -5039,20 +5039,64 @@ var csv_sync = (function (exports) { return dest; }; - const Generator = function(options = {}){ + const init_state$1 = (options) => { + // State + return { + start_time: options.duration ? Date.now() : null, + fixed_size_buffer: '', + count_written: 0, + count_created: 0, + }; + }; + + // Generate a random number between 0 and 1 with 2 decimals. The function is idempotent if it detect the "seed" option. + const random = function(options={}){ + if(options.seed){ + return options.seed = options.seed * Math.PI * 100 % 100 / 100; + }else { + return Math.random(); + } + }; + + const types = { + // Generate an ASCII value. + ascii: function({options}){ + const column = []; + const nb_chars = Math.ceil(random(options) * options.maxWordLength); + for(let i=0; i { // Convert Stream Readable options if underscored - if(options.high_water_mark){ - options.highWaterMark = options.high_water_mark; + if(opts.high_water_mark){ + opts.highWaterMark = opts.high_water_mark; } - if(options.object_mode){ - options.objectMode = options.object_mode; + if(opts.object_mode){ + opts.objectMode = opts.object_mode; } - // Call parent constructor - Stream.Readable.call(this, options); // Clone and camelize options - this.options = {}; - for(const k in options){ - this.options[Generator.camelize(k)] = options[k]; + const options = {}; + for(const k in opts){ + options[camelize(k)] = opts[k]; } // Normalize options const dft = { @@ -5070,110 +5114,100 @@ var csv_sync = (function (exports) { sleep: 0, }; for(const k in dft){ - if(this.options[k] === undefined){ - this.options[k] = dft[k]; + if(options[k] === undefined){ + options[k] = dft[k]; } } // Default values - if(this.options.eof === true){ - this.options.eof = this.options.rowDelimiter; + if(options.eof === true){ + options.eof = options.rowDelimiter; } - // State - this._ = { - start_time: this.options.duration ? Date.now() : null, - fixed_size_buffer: '', - count_written: 0, - count_created: 0, - }; - if(typeof this.options.columns === 'number'){ - this.options.columns = new Array(this.options.columns); + if(typeof options.columns === 'number'){ + options.columns = new Array(options.columns); } - const accepted_header_types = Object.keys(Generator).filter((t) => (!['super_', 'camelize'].includes(t))); - for(let i = 0; i < this.options.columns.length; i++){ - const v = this.options.columns[i] || 'ascii'; + const accepted_header_types = Object.keys(types).filter((t) => (!['super_', 'camelize'].includes(t))); + for(let i = 0; i < options.columns.length; i++){ + const v = options.columns[i] || 'ascii'; if(typeof v === 'string'){ if(!accepted_header_types.includes(v)){ throw Error(`Invalid column type: got "${v}", default values are ${JSON.stringify(accepted_header_types)}`); } - this.options.columns[i] = Generator[v]; + options.columns[i] = types[v]; } } - return this; + return options; }; - util.inherits(Generator, Stream.Readable); - // Generate a random number between 0 and 1 with 2 decimals. The function is idempotent if it detect the "seed" option. - Generator.prototype.random = function(){ - if(this.options.seed){ - return this.options.seed = this.options.seed * Math.PI * 100 % 100 / 100; - }else { - return Math.random(); - } - }; - // Stop the generation. - Generator.prototype.end = function(){ - this.push(null); - }; - // Put new data into the read queue. - Generator.prototype._read = function(size){ + const read = (options, state, size, push, close) => { // Already started const data = []; - let length = this._.fixed_size_buffer.length; + let length = state.fixed_size_buffer.length; if(length !== 0){ - data.push(this._.fixed_size_buffer); + data.push(state.fixed_size_buffer); } // eslint-disable-next-line while(true){ // Time for some rest: flush first and stop later - if((this._.count_created === this.options.length) || (this.options.end && Date.now() > this.options.end) || (this.options.duration && Date.now() > this._.start_time + this.options.duration)){ + if((state.count_created === options.length) || (options.end && Date.now() > options.end) || (options.duration && Date.now() > state.start_time + options.duration)){ // Flush if(data.length){ - if(this.options.objectMode){ + if(options.objectMode){ for(const record of data){ - this.__push(record); + push(record); } }else { - this.__push(data.join('') + (this.options.eof ? this.options.eof : '')); + push(data.join('') + (options.eof ? options.eof : '')); } - this._.end = true; + state.end = true; }else { - this.push(null); + close(); } return; } // Create the record let record = []; let recordLength; - this.options.columns.forEach((fn) => { - record.push(fn(this)); + options.columns.forEach((fn) => { + const result = fn({options: options, state: state}); + const type = typeof result; + if(result !== null && type !== 'string' && type !== 'number'){ + throw Error([ + 'INVALID_VALUE:', + 'values returned by column function must be', + 'a string, a number or null,', + `got ${JSON.stringify(result)}` + ].join(' ')); + } + record.push(result); }); // Obtain record length - if(this.options.objectMode){ + if(options.objectMode){ recordLength = 0; // recordLength is currently equal to the number of columns // This is wrong and shall equal to 1 record only - for(const column of record) + for(const column of record){ recordLength += column.length; + } }else { // Stringify the record - record = (this._.count_created === 0 ? '' : this.options.rowDelimiter)+record.join(this.options.delimiter); + record = (state.count_created === 0 ? '' : options.rowDelimiter)+record.join(options.delimiter); recordLength = record.length; } - this._.count_created++; + state.count_created++; if(length + recordLength > size){ - if(this.options.objectMode){ + if(options.objectMode){ data.push(record); for(const record of data){ - this.__push(record); + push(record); } }else { - if(this.options.fixedSize){ - this._.fixed_size_buffer = record.substr(size - length); + if(options.fixedSize){ + state.fixed_size_buffer = record.substr(size - length); data.push(record.substr(0, size - length)); }else { data.push(record); } - this.__push(data.join('')); + push(data.join('')); } return; } @@ -5181,42 +5215,41 @@ var csv_sync = (function (exports) { data.push(record); } }; + + const Generator = function(options = {}){ + this.options = normalize_options$2(options); + // Call parent constructor + Stream.Readable.call(this, this.options); + this.state = init_state$1(this.options); + return this; + }; + util.inherits(Generator, Stream.Readable); + + // Stop the generation. + Generator.prototype.end = function(){ + this.push(null); + }; + // Put new data into the read queue. + Generator.prototype._read = function(size){ + const self = this; + read(this.options, this.state, size, function(chunk) { + self.__push(chunk); + }, function(){ + self.push(null); + }); + }; // Put new data into the read queue. Generator.prototype.__push = function(record){ + // console.log('push', record) const push = () => { - this._.count_written++; + this.state.count_written++; this.push(record); - if(this._.end === true){ + if(this.state.end === true){ return this.push(null); } }; this.options.sleep > 0 ? setTimeout(push, this.options.sleep) : push(); }; - // Generate an ASCII value. - Generator.ascii = function(gen){ - // Column - const column = []; - const nb_chars = Math.ceil(gen.random() * gen.options.maxWordLength); - for(let i=0; i delimiter.length), + // Skip if the remaining buffer can be escape sequence + options.quote !== null ? options.quote.length : 0, + ), + previousBuf: undefined, + quoting: false, + stop: false, + rawBuffer: new ResizeableBuffer(100), + record: [], + recordHasError: false, + record_length: 0, + recordDelimiterMaxLength: options.record_delimiter.length === 0 ? 2 : Math.max(...options.record_delimiter.map((v) => v.length)), + trimChars: [Buffer.from(' ', options.encoding)[0], Buffer.from('\t', options.encoding)[0]], + wasQuoting: false, + wasRowDelimiter: false + }; }; - class CsvError$1 extends Error { - constructor(code, message, options, ...contexts) { - if(Array.isArray(message)) message = message.join(' '); - super(message); - if(Error.captureStackTrace !== undefined){ - Error.captureStackTrace(this, CsvError$1); - } - this.code = code; - for(const context of contexts){ - for(const key in context){ - const value = context[key]; - this[key] = isBuffer$1(value) ? value.toString(options.encoding) : value == null ? value : JSON.parse(JSON.stringify(value)); - } - } - } - } - const underscore$1 = function(str){ return str.replace(/([A-Z])/g, function(_, match){ return '_' + match.toLowerCase(); }); }; - const isObject$1 = function(obj){ - return (typeof obj === 'object' && obj !== null && !Array.isArray(obj)); - }; - - const isRecordEmpty = function(record){ - return record.every((field) => field == null || field.toString && field.toString().trim() === ''); - }; - - const normalizeColumnsArray = function(columns){ - const normalizedColumns = []; - for(let i = 0, l = columns.length; i < l; i++){ - const column = columns[i]; - if(column === undefined || column === null || column === false){ - normalizedColumns[i] = { disabled: true }; - }else if(typeof column === 'string'){ - normalizedColumns[i] = { name: column }; - }else if(isObject$1(column)){ - if(typeof column.name !== 'string'){ - throw new CsvError$1('CSV_OPTION_COLUMNS_MISSING_NAME', [ - 'Option columns missing name:', - `property "name" is required at position ${i}`, - 'when column is an object literal' - ]); - } - normalizedColumns[i] = column; - }else { - throw new CsvError$1('CSV_INVALID_COLUMN_DEFINITION', [ - 'Invalid column definition:', - 'expect a string or a literal object,', - `got ${JSON.stringify(column)} at position ${i}` - ]); - } - } - return normalizedColumns; - }; - - class Parser extends Transform { - constructor(opts = {}){ - super({...{readableObjectMode: true}, ...opts, encoding: null}); - this.__originalOptions = opts; - this.__normalizeOptions(opts); - } - __normalizeOptions(opts){ - const options = {}; - // Merge with user options - for(const opt in opts){ - options[underscore$1(opt)] = opts[opt]; - } - // Normalize option `encoding` - // Note: defined first because other options depends on it - // to convert chars/strings into buffers. - if(options.encoding === undefined || options.encoding === true){ - options.encoding = 'utf8'; - }else if(options.encoding === null || options.encoding === false){ - options.encoding = null; - }else if(typeof options.encoding !== 'string' && options.encoding !== null){ - throw new CsvError$1('CSV_INVALID_OPTION_ENCODING', [ - 'Invalid option encoding:', - 'encoding must be a string or null to return a buffer,', - `got ${JSON.stringify(options.encoding)}` - ], options); - } - // Normalize option `bom` - if(options.bom === undefined || options.bom === null || options.bom === false){ - options.bom = false; - }else if(options.bom !== true){ - throw new CsvError$1('CSV_INVALID_OPTION_BOM', [ - 'Invalid option bom:', 'bom must be true,', - `got ${JSON.stringify(options.bom)}` - ], options); - } - // Normalize option `cast` - let fnCastField = null; - if(options.cast === undefined || options.cast === null || options.cast === false || options.cast === ''){ - options.cast = undefined; - }else if(typeof options.cast === 'function'){ - fnCastField = options.cast; - options.cast = true; - }else if(options.cast !== true){ - throw new CsvError$1('CSV_INVALID_OPTION_CAST', [ - 'Invalid option cast:', 'cast must be true or a function,', - `got ${JSON.stringify(options.cast)}` - ], options); - } - // Normalize option `cast_date` - if(options.cast_date === undefined || options.cast_date === null || options.cast_date === false || options.cast_date === ''){ - options.cast_date = false; - }else if(options.cast_date === true){ - options.cast_date = function(value){ - const date = Date.parse(value); - return !isNaN(date) ? new Date(date) : value; - }; - }else { - throw new CsvError$1('CSV_INVALID_OPTION_CAST_DATE', [ - 'Invalid option cast_date:', 'cast_date must be true or a function,', - `got ${JSON.stringify(options.cast_date)}` - ], options); - } - // Normalize option `columns` - let fnFirstLineToHeaders = null; - if(options.columns === true){ - // Fields in the first line are converted as-is to columns - fnFirstLineToHeaders = undefined; - }else if(typeof options.columns === 'function'){ - fnFirstLineToHeaders = options.columns; - options.columns = true; - }else if(Array.isArray(options.columns)){ - options.columns = normalizeColumnsArray(options.columns); - }else if(options.columns === undefined || options.columns === null || options.columns === false){ - options.columns = false; - }else { - throw new CsvError$1('CSV_INVALID_OPTION_COLUMNS', [ - 'Invalid option columns:', - 'expect an array, a function or true,', - `got ${JSON.stringify(options.columns)}` + const normalize_options$1 = function(opts){ + const options = {}; + // Merge with user options + for(const opt in opts){ + options[underscore$1(opt)] = opts[opt]; + } + // Normalize option `encoding` + // Note: defined first because other options depends on it + // to convert chars/strings into buffers. + if(options.encoding === undefined || options.encoding === true){ + options.encoding = 'utf8'; + }else if(options.encoding === null || options.encoding === false){ + options.encoding = null; + }else if(typeof options.encoding !== 'string' && options.encoding !== null){ + throw new CsvError$1('CSV_INVALID_OPTION_ENCODING', [ + 'Invalid option encoding:', + 'encoding must be a string or null to return a buffer,', + `got ${JSON.stringify(options.encoding)}` + ], options); + } + // Normalize option `bom` + if(options.bom === undefined || options.bom === null || options.bom === false){ + options.bom = false; + }else if(options.bom !== true){ + throw new CsvError$1('CSV_INVALID_OPTION_BOM', [ + 'Invalid option bom:', 'bom must be true,', + `got ${JSON.stringify(options.bom)}` + ], options); + } + // Normalize option `cast` + options.cast_function = null; + if(options.cast === undefined || options.cast === null || options.cast === false || options.cast === ''){ + options.cast = undefined; + }else if(typeof options.cast === 'function'){ + options.cast_function = options.cast; + options.cast = true; + }else if(options.cast !== true){ + throw new CsvError$1('CSV_INVALID_OPTION_CAST', [ + 'Invalid option cast:', 'cast must be true or a function,', + `got ${JSON.stringify(options.cast)}` + ], options); + } + // Normalize option `cast_date` + if(options.cast_date === undefined || options.cast_date === null || options.cast_date === false || options.cast_date === ''){ + options.cast_date = false; + }else if(options.cast_date === true){ + options.cast_date = function(value){ + const date = Date.parse(value); + return !isNaN(date) ? new Date(date) : value; + }; + }else { + throw new CsvError$1('CSV_INVALID_OPTION_CAST_DATE', [ + 'Invalid option cast_date:', 'cast_date must be true or a function,', + `got ${JSON.stringify(options.cast_date)}` + ], options); + } + // Normalize option `columns` + options.cast_first_line_to_header = null; + if(options.columns === true){ + // Fields in the first line are converted as-is to columns + options.cast_first_line_to_header = undefined; + }else if(typeof options.columns === 'function'){ + options.cast_first_line_to_header = options.columns; + options.columns = true; + }else if(Array.isArray(options.columns)){ + options.columns = normalize_columns_array(options.columns); + }else if(options.columns === undefined || options.columns === null || options.columns === false){ + options.columns = false; + }else { + throw new CsvError$1('CSV_INVALID_OPTION_COLUMNS', [ + 'Invalid option columns:', + 'expect an array, a function or true,', + `got ${JSON.stringify(options.columns)}` + ], options); + } + // Normalize option `group_columns_by_name` + if(options.group_columns_by_name === undefined || options.group_columns_by_name === null || options.group_columns_by_name === false){ + options.group_columns_by_name = false; + }else if(options.group_columns_by_name !== true){ + throw new CsvError$1('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ + 'Invalid option group_columns_by_name:', + 'expect an boolean,', + `got ${JSON.stringify(options.group_columns_by_name)}` + ], options); + }else if(options.columns === false){ + throw new CsvError$1('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ + 'Invalid option group_columns_by_name:', + 'the `columns` mode must be activated.' + ], options); + } + // Normalize option `comment` + if(options.comment === undefined || options.comment === null || options.comment === false || options.comment === ''){ + options.comment = null; + }else { + if(typeof options.comment === 'string'){ + options.comment = Buffer.from(options.comment, options.encoding); + } + if(!isBuffer$1(options.comment)){ + throw new CsvError$1('CSV_INVALID_OPTION_COMMENT', [ + 'Invalid option comment:', + 'comment must be a buffer or a string,', + `got ${JSON.stringify(options.comment)}` ], options); } - // Normalize option `group_columns_by_name` - if(options.group_columns_by_name === undefined || options.group_columns_by_name === null || options.group_columns_by_name === false){ - options.group_columns_by_name = false; - }else if(options.group_columns_by_name !== true){ - throw new CsvError$1('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ - 'Invalid option group_columns_by_name:', - 'expect an boolean,', - `got ${JSON.stringify(options.group_columns_by_name)}` - ], options); - }else if(options.columns === false){ - throw new CsvError$1('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ - 'Invalid option group_columns_by_name:', - 'the `columns` mode must be activated.' - ], options); + } + // Normalize option `delimiter` + const delimiter_json = JSON.stringify(options.delimiter); + if(!Array.isArray(options.delimiter)) options.delimiter = [options.delimiter]; + if(options.delimiter.length === 0){ + throw new CsvError$1('CSV_INVALID_OPTION_DELIMITER', [ + 'Invalid option delimiter:', + 'delimiter must be a non empty string or buffer or array of string|buffer,', + `got ${delimiter_json}` + ], options); + } + options.delimiter = options.delimiter.map(function(delimiter){ + if(delimiter === undefined || delimiter === null || delimiter === false){ + return Buffer.from(',', options.encoding); } - // Normalize option `comment` - if(options.comment === undefined || options.comment === null || options.comment === false || options.comment === ''){ - options.comment = null; - }else { - if(typeof options.comment === 'string'){ - options.comment = Buffer.from(options.comment, options.encoding); - } - if(!isBuffer$1(options.comment)){ - throw new CsvError$1('CSV_INVALID_OPTION_COMMENT', [ - 'Invalid option comment:', - 'comment must be a buffer or a string,', - `got ${JSON.stringify(options.comment)}` - ], options); - } + if(typeof delimiter === 'string'){ + delimiter = Buffer.from(delimiter, options.encoding); } - // Normalize option `delimiter` - const delimiter_json = JSON.stringify(options.delimiter); - if(!Array.isArray(options.delimiter)) options.delimiter = [options.delimiter]; - if(options.delimiter.length === 0){ + if(!isBuffer$1(delimiter) || delimiter.length === 0){ throw new CsvError$1('CSV_INVALID_OPTION_DELIMITER', [ 'Invalid option delimiter:', 'delimiter must be a non empty string or buffer or array of string|buffer,', `got ${delimiter_json}` ], options); } - options.delimiter = options.delimiter.map(function(delimiter){ - if(delimiter === undefined || delimiter === null || delimiter === false){ - return Buffer.from(',', options.encoding); - } - if(typeof delimiter === 'string'){ - delimiter = Buffer.from(delimiter, options.encoding); - } - if(!isBuffer$1(delimiter) || delimiter.length === 0){ - throw new CsvError$1('CSV_INVALID_OPTION_DELIMITER', [ - 'Invalid option delimiter:', - 'delimiter must be a non empty string or buffer or array of string|buffer,', - `got ${delimiter_json}` - ], options); - } - return delimiter; - }); - // Normalize option `escape` - if(options.escape === undefined || options.escape === true){ - options.escape = Buffer.from('"', options.encoding); - }else if(typeof options.escape === 'string'){ - options.escape = Buffer.from(options.escape, options.encoding); - }else if (options.escape === null || options.escape === false){ - options.escape = null; - } - if(options.escape !== null){ - if(!isBuffer$1(options.escape)){ - throw new Error(`Invalid Option: escape must be a buffer, a string or a boolean, got ${JSON.stringify(options.escape)}`); - } + return delimiter; + }); + // Normalize option `escape` + if(options.escape === undefined || options.escape === true){ + options.escape = Buffer.from('"', options.encoding); + }else if(typeof options.escape === 'string'){ + options.escape = Buffer.from(options.escape, options.encoding); + }else if (options.escape === null || options.escape === false){ + options.escape = null; + } + if(options.escape !== null){ + if(!isBuffer$1(options.escape)){ + throw new Error(`Invalid Option: escape must be a buffer, a string or a boolean, got ${JSON.stringify(options.escape)}`); + } + } + // Normalize option `from` + if(options.from === undefined || options.from === null){ + options.from = 1; + }else { + if(typeof options.from === 'string' && /\d+/.test(options.from)){ + options.from = parseInt(options.from); } - // Normalize option `from` - if(options.from === undefined || options.from === null){ - options.from = 1; - }else { - if(typeof options.from === 'string' && /\d+/.test(options.from)){ - options.from = parseInt(options.from); - } - if(Number.isInteger(options.from)){ - if(options.from < 0){ - throw new Error(`Invalid Option: from must be a positive integer, got ${JSON.stringify(opts.from)}`); - } - }else { - throw new Error(`Invalid Option: from must be an integer, got ${JSON.stringify(options.from)}`); + if(Number.isInteger(options.from)){ + if(options.from < 0){ + throw new Error(`Invalid Option: from must be a positive integer, got ${JSON.stringify(opts.from)}`); } - } - // Normalize option `from_line` - if(options.from_line === undefined || options.from_line === null){ - options.from_line = 1; }else { - if(typeof options.from_line === 'string' && /\d+/.test(options.from_line)){ - options.from_line = parseInt(options.from_line); - } - if(Number.isInteger(options.from_line)){ - if(options.from_line <= 0){ - throw new Error(`Invalid Option: from_line must be a positive integer greater than 0, got ${JSON.stringify(opts.from_line)}`); - } - }else { - throw new Error(`Invalid Option: from_line must be an integer, got ${JSON.stringify(opts.from_line)}`); - } - } - // Normalize options `ignore_last_delimiters` - if(options.ignore_last_delimiters === undefined || options.ignore_last_delimiters === null){ - options.ignore_last_delimiters = false; - }else if(typeof options.ignore_last_delimiters === 'number'){ - options.ignore_last_delimiters = Math.floor(options.ignore_last_delimiters); - if(options.ignore_last_delimiters === 0){ - options.ignore_last_delimiters = false; - } - }else if(typeof options.ignore_last_delimiters !== 'boolean'){ - throw new CsvError$1('CSV_INVALID_OPTION_IGNORE_LAST_DELIMITERS', [ - 'Invalid option `ignore_last_delimiters`:', - 'the value must be a boolean value or an integer,', - `got ${JSON.stringify(options.ignore_last_delimiters)}` - ], options); + throw new Error(`Invalid Option: from must be an integer, got ${JSON.stringify(options.from)}`); } - if(options.ignore_last_delimiters === true && options.columns === false){ - throw new CsvError$1('CSV_IGNORE_LAST_DELIMITERS_REQUIRES_COLUMNS', [ - 'The option `ignore_last_delimiters`', - 'requires the activation of the `columns` option' - ], options); + } + // Normalize option `from_line` + if(options.from_line === undefined || options.from_line === null){ + options.from_line = 1; + }else { + if(typeof options.from_line === 'string' && /\d+/.test(options.from_line)){ + options.from_line = parseInt(options.from_line); } - // Normalize option `info` - if(options.info === undefined || options.info === null || options.info === false){ - options.info = false; - }else if(options.info !== true){ - throw new Error(`Invalid Option: info must be true, got ${JSON.stringify(options.info)}`); - } - // Normalize option `max_record_size` - if(options.max_record_size === undefined || options.max_record_size === null || options.max_record_size === false){ - options.max_record_size = 0; - }else if(Number.isInteger(options.max_record_size) && options.max_record_size >= 0);else if(typeof options.max_record_size === 'string' && /\d+/.test(options.max_record_size)){ - options.max_record_size = parseInt(options.max_record_size); - }else { - throw new Error(`Invalid Option: max_record_size must be a positive integer, got ${JSON.stringify(options.max_record_size)}`); - } - // Normalize option `objname` - if(options.objname === undefined || options.objname === null || options.objname === false){ - options.objname = undefined; - }else if(isBuffer$1(options.objname)){ - if(options.objname.length === 0){ - throw new Error(`Invalid Option: objname must be a non empty buffer`); - } - if(options.encoding === null);else { - options.objname = options.objname.toString(options.encoding); - } - }else if(typeof options.objname === 'string'){ - if(options.objname.length === 0){ - throw new Error(`Invalid Option: objname must be a non empty string`); - } - // Great, nothing to do - }else if(typeof options.objname === 'number');else { - throw new Error(`Invalid Option: objname must be a string or a buffer, got ${options.objname}`); - } - if(options.objname !== undefined){ - if(typeof options.objname === 'number'){ - if(options.columns !== false){ - throw Error('Invalid Option: objname index cannot be combined with columns or be defined as a field'); - } - }else { // A string or a buffer - if(options.columns === false){ - throw Error('Invalid Option: objname field must be combined with columns or be defined as an index'); - } + if(Number.isInteger(options.from_line)){ + if(options.from_line <= 0){ + throw new Error(`Invalid Option: from_line must be a positive integer greater than 0, got ${JSON.stringify(opts.from_line)}`); } + }else { + throw new Error(`Invalid Option: from_line must be an integer, got ${JSON.stringify(opts.from_line)}`); } - // Normalize option `on_record` - if(options.on_record === undefined || options.on_record === null){ - options.on_record = undefined; - }else if(typeof options.on_record !== 'function'){ - throw new CsvError$1('CSV_INVALID_OPTION_ON_RECORD', [ - 'Invalid option `on_record`:', - 'expect a function,', - `got ${JSON.stringify(options.on_record)}` - ], options); + } + // Normalize options `ignore_last_delimiters` + if(options.ignore_last_delimiters === undefined || options.ignore_last_delimiters === null){ + options.ignore_last_delimiters = false; + }else if(typeof options.ignore_last_delimiters === 'number'){ + options.ignore_last_delimiters = Math.floor(options.ignore_last_delimiters); + if(options.ignore_last_delimiters === 0){ + options.ignore_last_delimiters = false; } - // Normalize option `quote` - if(options.quote === null || options.quote === false || options.quote === ''){ - options.quote = null; - }else { - if(options.quote === undefined || options.quote === true){ - options.quote = Buffer.from('"', options.encoding); - }else if(typeof options.quote === 'string'){ - options.quote = Buffer.from(options.quote, options.encoding); + }else if(typeof options.ignore_last_delimiters !== 'boolean'){ + throw new CsvError$1('CSV_INVALID_OPTION_IGNORE_LAST_DELIMITERS', [ + 'Invalid option `ignore_last_delimiters`:', + 'the value must be a boolean value or an integer,', + `got ${JSON.stringify(options.ignore_last_delimiters)}` + ], options); + } + if(options.ignore_last_delimiters === true && options.columns === false){ + throw new CsvError$1('CSV_IGNORE_LAST_DELIMITERS_REQUIRES_COLUMNS', [ + 'The option `ignore_last_delimiters`', + 'requires the activation of the `columns` option' + ], options); + } + // Normalize option `info` + if(options.info === undefined || options.info === null || options.info === false){ + options.info = false; + }else if(options.info !== true){ + throw new Error(`Invalid Option: info must be true, got ${JSON.stringify(options.info)}`); + } + // Normalize option `max_record_size` + if(options.max_record_size === undefined || options.max_record_size === null || options.max_record_size === false){ + options.max_record_size = 0; + }else if(Number.isInteger(options.max_record_size) && options.max_record_size >= 0);else if(typeof options.max_record_size === 'string' && /\d+/.test(options.max_record_size)){ + options.max_record_size = parseInt(options.max_record_size); + }else { + throw new Error(`Invalid Option: max_record_size must be a positive integer, got ${JSON.stringify(options.max_record_size)}`); + } + // Normalize option `objname` + if(options.objname === undefined || options.objname === null || options.objname === false){ + options.objname = undefined; + }else if(isBuffer$1(options.objname)){ + if(options.objname.length === 0){ + throw new Error(`Invalid Option: objname must be a non empty buffer`); + } + if(options.encoding === null);else { + options.objname = options.objname.toString(options.encoding); + } + }else if(typeof options.objname === 'string'){ + if(options.objname.length === 0){ + throw new Error(`Invalid Option: objname must be a non empty string`); + } + // Great, nothing to do + }else if(typeof options.objname === 'number');else { + throw new Error(`Invalid Option: objname must be a string or a buffer, got ${options.objname}`); + } + if(options.objname !== undefined){ + if(typeof options.objname === 'number'){ + if(options.columns !== false){ + throw Error('Invalid Option: objname index cannot be combined with columns or be defined as a field'); } - if(!isBuffer$1(options.quote)){ - throw new Error(`Invalid Option: quote must be a buffer or a string, got ${JSON.stringify(options.quote)}`); + }else { // A string or a buffer + if(options.columns === false){ + throw Error('Invalid Option: objname field must be combined with columns or be defined as an index'); } } - // Normalize option `raw` - if(options.raw === undefined || options.raw === null || options.raw === false){ - options.raw = false; - }else if(options.raw !== true){ - throw new Error(`Invalid Option: raw must be true, got ${JSON.stringify(options.raw)}`); - } - // Normalize option `record_delimiter` - if(options.record_delimiter === undefined){ - options.record_delimiter = []; - }else if(typeof options.record_delimiter === 'string' || isBuffer$1(options.record_delimiter)){ - if(options.record_delimiter.length === 0){ - throw new CsvError$1('CSV_INVALID_OPTION_RECORD_DELIMITER', [ - 'Invalid option `record_delimiter`:', - 'value must be a non empty string or buffer,', - `got ${JSON.stringify(options.record_delimiter)}` - ], options); - } - options.record_delimiter = [options.record_delimiter]; - }else if(!Array.isArray(options.record_delimiter)){ + } + // Normalize option `on_record` + if(options.on_record === undefined || options.on_record === null){ + options.on_record = undefined; + }else if(typeof options.on_record !== 'function'){ + throw new CsvError$1('CSV_INVALID_OPTION_ON_RECORD', [ + 'Invalid option `on_record`:', + 'expect a function,', + `got ${JSON.stringify(options.on_record)}` + ], options); + } + // Normalize option `quote` + if(options.quote === null || options.quote === false || options.quote === ''){ + options.quote = null; + }else { + if(options.quote === undefined || options.quote === true){ + options.quote = Buffer.from('"', options.encoding); + }else if(typeof options.quote === 'string'){ + options.quote = Buffer.from(options.quote, options.encoding); + } + if(!isBuffer$1(options.quote)){ + throw new Error(`Invalid Option: quote must be a buffer or a string, got ${JSON.stringify(options.quote)}`); + } + } + // Normalize option `raw` + if(options.raw === undefined || options.raw === null || options.raw === false){ + options.raw = false; + }else if(options.raw !== true){ + throw new Error(`Invalid Option: raw must be true, got ${JSON.stringify(options.raw)}`); + } + // Normalize option `record_delimiter` + if(options.record_delimiter === undefined){ + options.record_delimiter = []; + }else if(typeof options.record_delimiter === 'string' || isBuffer$1(options.record_delimiter)){ + if(options.record_delimiter.length === 0){ throw new CsvError$1('CSV_INVALID_OPTION_RECORD_DELIMITER', [ 'Invalid option `record_delimiter`:', - 'value must be a string, a buffer or array of string|buffer,', + 'value must be a non empty string or buffer,', `got ${JSON.stringify(options.record_delimiter)}` ], options); } - options.record_delimiter = options.record_delimiter.map(function(rd, i){ - if(typeof rd !== 'string' && ! isBuffer$1(rd)){ - throw new CsvError$1('CSV_INVALID_OPTION_RECORD_DELIMITER', [ - 'Invalid option `record_delimiter`:', - 'value must be a string, a buffer or array of string|buffer', - `at index ${i},`, - `got ${JSON.stringify(rd)}` - ], options); - }else if(rd.length === 0){ - throw new CsvError$1('CSV_INVALID_OPTION_RECORD_DELIMITER', [ - 'Invalid option `record_delimiter`:', - 'value must be a non empty string or buffer', - `at index ${i},`, - `got ${JSON.stringify(rd)}` - ], options); - } - if(typeof rd === 'string'){ - rd = Buffer.from(rd, options.encoding); - } - return rd; - }); - // Normalize option `relax_column_count` - if(typeof options.relax_column_count === 'boolean');else if(options.relax_column_count === undefined || options.relax_column_count === null){ - options.relax_column_count = false; - }else { - throw new Error(`Invalid Option: relax_column_count must be a boolean, got ${JSON.stringify(options.relax_column_count)}`); - } - if(typeof options.relax_column_count_less === 'boolean');else if(options.relax_column_count_less === undefined || options.relax_column_count_less === null){ - options.relax_column_count_less = false; - }else { - throw new Error(`Invalid Option: relax_column_count_less must be a boolean, got ${JSON.stringify(options.relax_column_count_less)}`); - } - if(typeof options.relax_column_count_more === 'boolean');else if(options.relax_column_count_more === undefined || options.relax_column_count_more === null){ - options.relax_column_count_more = false; - }else { - throw new Error(`Invalid Option: relax_column_count_more must be a boolean, got ${JSON.stringify(options.relax_column_count_more)}`); - } - // Normalize option `relax_quotes` - if(typeof options.relax_quotes === 'boolean');else if(options.relax_quotes === undefined || options.relax_quotes === null){ - options.relax_quotes = false; - }else { - throw new Error(`Invalid Option: relax_quotes must be a boolean, got ${JSON.stringify(options.relax_quotes)}`); + options.record_delimiter = [options.record_delimiter]; + }else if(!Array.isArray(options.record_delimiter)){ + throw new CsvError$1('CSV_INVALID_OPTION_RECORD_DELIMITER', [ + 'Invalid option `record_delimiter`:', + 'value must be a string, a buffer or array of string|buffer,', + `got ${JSON.stringify(options.record_delimiter)}` + ], options); + } + options.record_delimiter = options.record_delimiter.map(function(rd, i){ + if(typeof rd !== 'string' && ! isBuffer$1(rd)){ + throw new CsvError$1('CSV_INVALID_OPTION_RECORD_DELIMITER', [ + 'Invalid option `record_delimiter`:', + 'value must be a string, a buffer or array of string|buffer', + `at index ${i},`, + `got ${JSON.stringify(rd)}` + ], options); + }else if(rd.length === 0){ + throw new CsvError$1('CSV_INVALID_OPTION_RECORD_DELIMITER', [ + 'Invalid option `record_delimiter`:', + 'value must be a non empty string or buffer', + `at index ${i},`, + `got ${JSON.stringify(rd)}` + ], options); } - // Normalize option `skip_empty_lines` - if(typeof options.skip_empty_lines === 'boolean');else if(options.skip_empty_lines === undefined || options.skip_empty_lines === null){ - options.skip_empty_lines = false; - }else { - throw new Error(`Invalid Option: skip_empty_lines must be a boolean, got ${JSON.stringify(options.skip_empty_lines)}`); + if(typeof rd === 'string'){ + rd = Buffer.from(rd, options.encoding); } - // Normalize option `skip_records_with_empty_values` - if(typeof options.skip_records_with_empty_values === 'boolean');else if(options.skip_records_with_empty_values === undefined || options.skip_records_with_empty_values === null){ - options.skip_records_with_empty_values = false; - }else { - throw new Error(`Invalid Option: skip_records_with_empty_values must be a boolean, got ${JSON.stringify(options.skip_records_with_empty_values)}`); + return rd; + }); + // Normalize option `relax_column_count` + if(typeof options.relax_column_count === 'boolean');else if(options.relax_column_count === undefined || options.relax_column_count === null){ + options.relax_column_count = false; + }else { + throw new Error(`Invalid Option: relax_column_count must be a boolean, got ${JSON.stringify(options.relax_column_count)}`); + } + if(typeof options.relax_column_count_less === 'boolean');else if(options.relax_column_count_less === undefined || options.relax_column_count_less === null){ + options.relax_column_count_less = false; + }else { + throw new Error(`Invalid Option: relax_column_count_less must be a boolean, got ${JSON.stringify(options.relax_column_count_less)}`); + } + if(typeof options.relax_column_count_more === 'boolean');else if(options.relax_column_count_more === undefined || options.relax_column_count_more === null){ + options.relax_column_count_more = false; + }else { + throw new Error(`Invalid Option: relax_column_count_more must be a boolean, got ${JSON.stringify(options.relax_column_count_more)}`); + } + // Normalize option `relax_quotes` + if(typeof options.relax_quotes === 'boolean');else if(options.relax_quotes === undefined || options.relax_quotes === null){ + options.relax_quotes = false; + }else { + throw new Error(`Invalid Option: relax_quotes must be a boolean, got ${JSON.stringify(options.relax_quotes)}`); + } + // Normalize option `skip_empty_lines` + if(typeof options.skip_empty_lines === 'boolean');else if(options.skip_empty_lines === undefined || options.skip_empty_lines === null){ + options.skip_empty_lines = false; + }else { + throw new Error(`Invalid Option: skip_empty_lines must be a boolean, got ${JSON.stringify(options.skip_empty_lines)}`); + } + // Normalize option `skip_records_with_empty_values` + if(typeof options.skip_records_with_empty_values === 'boolean');else if(options.skip_records_with_empty_values === undefined || options.skip_records_with_empty_values === null){ + options.skip_records_with_empty_values = false; + }else { + throw new Error(`Invalid Option: skip_records_with_empty_values must be a boolean, got ${JSON.stringify(options.skip_records_with_empty_values)}`); + } + // Normalize option `skip_records_with_error` + if(typeof options.skip_records_with_error === 'boolean');else if(options.skip_records_with_error === undefined || options.skip_records_with_error === null){ + options.skip_records_with_error = false; + }else { + throw new Error(`Invalid Option: skip_records_with_error must be a boolean, got ${JSON.stringify(options.skip_records_with_error)}`); + } + // Normalize option `rtrim` + if(options.rtrim === undefined || options.rtrim === null || options.rtrim === false){ + options.rtrim = false; + }else if(options.rtrim !== true){ + throw new Error(`Invalid Option: rtrim must be a boolean, got ${JSON.stringify(options.rtrim)}`); + } + // Normalize option `ltrim` + if(options.ltrim === undefined || options.ltrim === null || options.ltrim === false){ + options.ltrim = false; + }else if(options.ltrim !== true){ + throw new Error(`Invalid Option: ltrim must be a boolean, got ${JSON.stringify(options.ltrim)}`); + } + // Normalize option `trim` + if(options.trim === undefined || options.trim === null || options.trim === false){ + options.trim = false; + }else if(options.trim !== true){ + throw new Error(`Invalid Option: trim must be a boolean, got ${JSON.stringify(options.trim)}`); + } + // Normalize options `trim`, `ltrim` and `rtrim` + if(options.trim === true && opts.ltrim !== false){ + options.ltrim = true; + }else if(options.ltrim !== true){ + options.ltrim = false; + } + if(options.trim === true && opts.rtrim !== false){ + options.rtrim = true; + }else if(options.rtrim !== true){ + options.rtrim = false; + } + // Normalize option `to` + if(options.to === undefined || options.to === null){ + options.to = -1; + }else { + if(typeof options.to === 'string' && /\d+/.test(options.to)){ + options.to = parseInt(options.to); } - // Normalize option `skip_records_with_error` - if(typeof options.skip_records_with_error === 'boolean');else if(options.skip_records_with_error === undefined || options.skip_records_with_error === null){ - options.skip_records_with_error = false; - }else { - throw new Error(`Invalid Option: skip_records_with_error must be a boolean, got ${JSON.stringify(options.skip_records_with_error)}`); - } - // Normalize option `rtrim` - if(options.rtrim === undefined || options.rtrim === null || options.rtrim === false){ - options.rtrim = false; - }else if(options.rtrim !== true){ - throw new Error(`Invalid Option: rtrim must be a boolean, got ${JSON.stringify(options.rtrim)}`); - } - // Normalize option `ltrim` - if(options.ltrim === undefined || options.ltrim === null || options.ltrim === false){ - options.ltrim = false; - }else if(options.ltrim !== true){ - throw new Error(`Invalid Option: ltrim must be a boolean, got ${JSON.stringify(options.ltrim)}`); - } - // Normalize option `trim` - if(options.trim === undefined || options.trim === null || options.trim === false){ - options.trim = false; - }else if(options.trim !== true){ - throw new Error(`Invalid Option: trim must be a boolean, got ${JSON.stringify(options.trim)}`); - } - // Normalize options `trim`, `ltrim` and `rtrim` - if(options.trim === true && opts.ltrim !== false){ - options.ltrim = true; - }else if(options.ltrim !== true){ - options.ltrim = false; - } - if(options.trim === true && opts.rtrim !== false){ - options.rtrim = true; - }else if(options.rtrim !== true){ - options.rtrim = false; - } - // Normalize option `to` - if(options.to === undefined || options.to === null){ - options.to = -1; - }else { - if(typeof options.to === 'string' && /\d+/.test(options.to)){ - options.to = parseInt(options.to); - } - if(Number.isInteger(options.to)){ - if(options.to <= 0){ - throw new Error(`Invalid Option: to must be a positive integer greater than 0, got ${JSON.stringify(opts.to)}`); - } - }else { - throw new Error(`Invalid Option: to must be an integer, got ${JSON.stringify(opts.to)}`); + if(Number.isInteger(options.to)){ + if(options.to <= 0){ + throw new Error(`Invalid Option: to must be a positive integer greater than 0, got ${JSON.stringify(opts.to)}`); } - } - // Normalize option `to_line` - if(options.to_line === undefined || options.to_line === null){ - options.to_line = -1; }else { - if(typeof options.to_line === 'string' && /\d+/.test(options.to_line)){ - options.to_line = parseInt(options.to_line); - } - if(Number.isInteger(options.to_line)){ - if(options.to_line <= 0){ - throw new Error(`Invalid Option: to_line must be a positive integer greater than 0, got ${JSON.stringify(opts.to_line)}`); - } - }else { - throw new Error(`Invalid Option: to_line must be an integer, got ${JSON.stringify(opts.to_line)}`); - } + throw new Error(`Invalid Option: to must be an integer, got ${JSON.stringify(opts.to)}`); } - this.info = { - bytes: 0, - comment_lines: 0, - empty_lines: 0, - invalid_field_length: 0, - lines: 1, - records: 0 - }; - this.options = options; - this.state = { - bomSkipped: false, - bufBytesStart: 0, - castField: fnCastField, - commenting: false, - // Current error encountered by a record - error: undefined, - enabled: options.from_line === 1, - escaping: false, - escapeIsQuote: isBuffer$1(options.escape) && isBuffer$1(options.quote) && Buffer.compare(options.escape, options.quote) === 0, - // columns can be `false`, `true`, `Array` - expectedRecordLength: Array.isArray(options.columns) ? options.columns.length : undefined, - field: new ResizeableBuffer(20), - firstLineToHeaders: fnFirstLineToHeaders, - needMoreDataSize: Math.max( - // Skip if the remaining buffer smaller than comment - options.comment !== null ? options.comment.length : 0, - // Skip if the remaining buffer can be delimiter - ...options.delimiter.map((delimiter) => delimiter.length), - // Skip if the remaining buffer can be escape sequence - options.quote !== null ? options.quote.length : 0, - ), - previousBuf: undefined, - quoting: false, - stop: false, - rawBuffer: new ResizeableBuffer(100), - record: [], - recordHasError: false, - record_length: 0, - recordDelimiterMaxLength: options.record_delimiter.length === 0 ? 2 : Math.max(...options.record_delimiter.map((v) => v.length)), - trimChars: [Buffer.from(' ', options.encoding)[0], Buffer.from('\t', options.encoding)[0]], - wasQuoting: false, - wasRowDelimiter: false - }; } - // Implementation of `Transform._transform` - _transform(buf, encoding, callback){ - if(this.state.stop === true){ - return; - } - const err = this.__parse(buf, false); - if(err !== undefined){ - this.state.stop = true; - } - callback(err); - } - // Implementation of `Transform._flush` - _flush(callback){ - if(this.state.stop === true){ - return; + // Normalize option `to_line` + if(options.to_line === undefined || options.to_line === null){ + options.to_line = -1; + }else { + if(typeof options.to_line === 'string' && /\d+/.test(options.to_line)){ + options.to_line = parseInt(options.to_line); } - const err = this.__parse(undefined, true); - callback(err); - } - // Central parser implementation - __parse(nextBuf, end){ - const {bom, comment, escape, from_line, ltrim, max_record_size, quote, raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options; - let {record_delimiter} = this.options; - const {bomSkipped, previousBuf, rawBuffer, escapeIsQuote} = this.state; - let buf; - if(previousBuf === undefined){ - if(nextBuf === undefined){ - // Handle empty string - this.push(null); - return; - }else { - buf = nextBuf; + if(Number.isInteger(options.to_line)){ + if(options.to_line <= 0){ + throw new Error(`Invalid Option: to_line must be a positive integer greater than 0, got ${JSON.stringify(opts.to_line)}`); } - }else if(previousBuf !== undefined && nextBuf === undefined){ - buf = previousBuf; }else { - buf = Buffer.concat([previousBuf, nextBuf]); - } - // Handle UTF BOM - if(bomSkipped === false){ - if(bom === false){ - this.state.bomSkipped = true; - }else if(buf.length < 3){ - // No enough data - if(end === false){ - // Wait for more data - this.state.previousBuf = buf; + throw new Error(`Invalid Option: to_line must be an integer, got ${JSON.stringify(opts.to_line)}`); + } + } + return options; + }; + + const isRecordEmpty = function(record){ + return record.every((field) => field == null || field.toString && field.toString().trim() === ''); + }; + + // white space characters + // https://en.wikipedia.org/wiki/Whitespace_character + // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions/Character_Classes#Types + // \f\n\r\t\v\u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff + const tab = 9; + const nl = 10; // \n, 0x0A in hexadecimal, 10 in decimal + const np = 12; + const cr = 13; // \r, 0x0D in hexadécimal, 13 in decimal + const space = 32; + const boms = { + // Note, the following are equals: + // Buffer.from("\ufeff") + // Buffer.from([239, 187, 191]) + // Buffer.from('EFBBBF', 'hex') + 'utf8': Buffer.from([239, 187, 191]), + // Note, the following are equals: + // Buffer.from "\ufeff", 'utf16le + // Buffer.from([255, 254]) + 'utf16le': Buffer.from([255, 254]) + }; + + const transform$1 = function(original_options = {}) { + const info = { + bytes: 0, + comment_lines: 0, + empty_lines: 0, + invalid_field_length: 0, + lines: 1, + records: 0 + }; + const options = normalize_options$1(original_options); + return { + info: info, + original_options: original_options, + options: options, + state: init_state(options), + __needMoreData: function(i, bufLen, end){ + if(end) return false; + const {quote} = this.options; + const {quoting, needMoreDataSize, recordDelimiterMaxLength} = this.state; + const numOfCharLeft = bufLen - i - 1; + const requiredLength = Math.max( + needMoreDataSize, + // Skip if the remaining buffer smaller than record delimiter + recordDelimiterMaxLength, + // Skip if the remaining buffer can be record delimiter following the closing quote + // 1 is for quote.length + quoting ? (quote.length + recordDelimiterMaxLength) : 0, + ); + return numOfCharLeft < requiredLength; + }, + // Central parser implementation + parse: function(nextBuf, end, push, close){ + const {bom, comment, escape, from_line, ltrim, max_record_size, quote, raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options; + let {record_delimiter} = this.options; + const {bomSkipped, previousBuf, rawBuffer, escapeIsQuote} = this.state; + let buf; + if(previousBuf === undefined){ + if(nextBuf === undefined){ + // Handle empty string + close(); return; + }else { + buf = nextBuf; } + }else if(previousBuf !== undefined && nextBuf === undefined){ + buf = previousBuf; }else { - for(const encoding in boms){ - if(boms[encoding].compare(buf, 0, boms[encoding].length) === 0){ - // Skip BOM - const bomLength = boms[encoding].length; - this.state.bufBytesStart += bomLength; - buf = buf.slice(bomLength); - // Renormalize original options with the new encoding - this.__normalizeOptions({...this.__originalOptions, encoding: encoding}); - break; + buf = Buffer.concat([previousBuf, nextBuf]); + } + // Handle UTF BOM + if(bomSkipped === false){ + if(bom === false){ + this.state.bomSkipped = true; + }else if(buf.length < 3){ + // No enough data + if(end === false){ + // Wait for more data + this.state.previousBuf = buf; + return; } + }else { + for(const encoding in boms){ + if(boms[encoding].compare(buf, 0, boms[encoding].length) === 0){ + // Skip BOM + const bomLength = boms[encoding].length; + this.state.bufBytesStart += bomLength; + buf = buf.slice(bomLength); + // Renormalize original options with the new encoding + this.options = normalize_options$1({...this.original_options, encoding: encoding}); + break; + } + } + this.state.bomSkipped = true; } - this.state.bomSkipped = true; - } - } - const bufLen = buf.length; - let pos; - for(pos = 0; pos < bufLen; pos++){ - // Ensure we get enough space to look ahead - // There should be a way to move this out of the loop - if(this.__needMoreData(pos, bufLen, end)){ - break; - } - if(this.state.wasRowDelimiter === true){ - this.info.lines++; - this.state.wasRowDelimiter = false; - } - if(to_line !== -1 && this.info.lines > to_line){ - this.state.stop = true; - this.push(null); - return; } - // Auto discovery of record_delimiter, unix, mac and windows supported - if(this.state.quoting === false && record_delimiter.length === 0){ - const record_delimiterCount = this.__autoDiscoverRecordDelimiter(buf, pos); - if(record_delimiterCount){ - record_delimiter = this.options.record_delimiter; + const bufLen = buf.length; + let pos; + for(pos = 0; pos < bufLen; pos++){ + // Ensure we get enough space to look ahead + // There should be a way to move this out of the loop + if(this.__needMoreData(pos, bufLen, end)){ + break; } - } - const chr = buf[pos]; - if(raw === true){ - rawBuffer.append(chr); - } - if((chr === cr || chr === nl) && this.state.wasRowDelimiter === false){ - this.state.wasRowDelimiter = true; - } - // Previous char was a valid escape char - // treat the current char as a regular char - if(this.state.escaping === true){ - this.state.escaping = false; - }else { - // Escape is only active inside quoted fields - // We are quoting, the char is an escape chr and there is a chr to escape - // if(escape !== null && this.state.quoting === true && chr === escape && pos + 1 < bufLen){ - if(escape !== null && this.state.quoting === true && this.__isEscape(buf, pos, chr) && pos + escape.length < bufLen){ - if(escapeIsQuote){ - if(this.__isQuote(buf, pos+escape.length)){ - this.state.escaping = true; - pos += escape.length - 1; - continue; - } - }else { - this.state.escaping = true; - pos += escape.length - 1; - continue; + if(this.state.wasRowDelimiter === true){ + this.info.lines++; + this.state.wasRowDelimiter = false; + } + if(to_line !== -1 && this.info.lines > to_line){ + this.state.stop = true; + close(); + return; + } + // Auto discovery of record_delimiter, unix, mac and windows supported + if(this.state.quoting === false && record_delimiter.length === 0){ + const record_delimiterCount = this.__autoDiscoverRecordDelimiter(buf, pos); + if(record_delimiterCount){ + record_delimiter = this.options.record_delimiter; } } - // Not currently escaping and chr is a quote - // TODO: need to compare bytes instead of single char - if(this.state.commenting === false && this.__isQuote(buf, pos)){ - if(this.state.quoting === true){ - const nextChr = buf[pos+quote.length]; - const isNextChrTrimable = rtrim && this.__isCharTrimable(nextChr); - const isNextChrComment = comment !== null && this.__compareBytes(comment, buf, pos+quote.length, nextChr); - const isNextChrDelimiter = this.__isDelimiter(buf, pos+quote.length, nextChr); - const isNextChrRecordDelimiter = record_delimiter.length === 0 ? this.__autoDiscoverRecordDelimiter(buf, pos+quote.length) : this.__isRecordDelimiter(nextChr, buf, pos+quote.length); - // Escape a quote - // Treat next char as a regular character - if(escape !== null && this.__isEscape(buf, pos, chr) && this.__isQuote(buf, pos + escape.length)){ + const chr = buf[pos]; + if(raw === true){ + rawBuffer.append(chr); + } + if((chr === cr || chr === nl) && this.state.wasRowDelimiter === false){ + this.state.wasRowDelimiter = true; + } + // Previous char was a valid escape char + // treat the current char as a regular char + if(this.state.escaping === true){ + this.state.escaping = false; + }else { + // Escape is only active inside quoted fields + // We are quoting, the char is an escape chr and there is a chr to escape + // if(escape !== null && this.state.quoting === true && chr === escape && pos + 1 < bufLen){ + if(escape !== null && this.state.quoting === true && this.__isEscape(buf, pos, chr) && pos + escape.length < bufLen){ + if(escapeIsQuote){ + if(this.__isQuote(buf, pos+escape.length)){ + this.state.escaping = true; + pos += escape.length - 1; + continue; + } + }else { + this.state.escaping = true; pos += escape.length - 1; - }else if(!nextChr || isNextChrDelimiter || isNextChrRecordDelimiter || isNextChrComment || isNextChrTrimable){ - this.state.quoting = false; - this.state.wasQuoting = true; - pos += quote.length - 1; continue; - }else if(relax_quotes === false){ - const err = this.__error( - new CsvError$1('CSV_INVALID_CLOSING_QUOTE', [ - 'Invalid Closing Quote:', - `got "${String.fromCharCode(nextChr)}"`, - `at line ${this.info.lines}`, - 'instead of delimiter, record delimiter, trimable character', - '(if activated) or comment', - ], this.options, this.__infoField()) - ); - if(err !== undefined) return err; - }else { - this.state.quoting = false; - this.state.wasQuoting = true; - this.state.field.prepend(quote); - pos += quote.length - 1; } - }else { - if(this.state.field.length !== 0){ - // In relax_quotes mode, treat opening quote preceded by chrs as regular - if(relax_quotes === false){ + } + // Not currently escaping and chr is a quote + // TODO: need to compare bytes instead of single char + if(this.state.commenting === false && this.__isQuote(buf, pos)){ + if(this.state.quoting === true){ + const nextChr = buf[pos+quote.length]; + const isNextChrTrimable = rtrim && this.__isCharTrimable(nextChr); + const isNextChrComment = comment !== null && this.__compareBytes(comment, buf, pos+quote.length, nextChr); + const isNextChrDelimiter = this.__isDelimiter(buf, pos+quote.length, nextChr); + const isNextChrRecordDelimiter = record_delimiter.length === 0 ? this.__autoDiscoverRecordDelimiter(buf, pos+quote.length) : this.__isRecordDelimiter(nextChr, buf, pos+quote.length); + // Escape a quote + // Treat next char as a regular character + if(escape !== null && this.__isEscape(buf, pos, chr) && this.__isQuote(buf, pos + escape.length)){ + pos += escape.length - 1; + }else if(!nextChr || isNextChrDelimiter || isNextChrRecordDelimiter || isNextChrComment || isNextChrTrimable){ + this.state.quoting = false; + this.state.wasQuoting = true; + pos += quote.length - 1; + continue; + }else if(relax_quotes === false){ const err = this.__error( - new CsvError$1('INVALID_OPENING_QUOTE', [ - 'Invalid Opening Quote:', - `a quote is found inside a field at line ${this.info.lines}`, - ], this.options, this.__infoField(), { - field: this.state.field, - }) + new CsvError$1('CSV_INVALID_CLOSING_QUOTE', [ + 'Invalid Closing Quote:', + `got "${String.fromCharCode(nextChr)}"`, + `at line ${this.info.lines}`, + 'instead of delimiter, record delimiter, trimable character', + '(if activated) or comment', + ], this.options, this.__infoField()) ); if(err !== undefined) return err; + }else { + this.state.quoting = false; + this.state.wasQuoting = true; + this.state.field.prepend(quote); + pos += quote.length - 1; } }else { - this.state.quoting = true; - pos += quote.length - 1; - continue; - } - } - } - if(this.state.quoting === false){ - const recordDelimiterLength = this.__isRecordDelimiter(chr, buf, pos); - if(recordDelimiterLength !== 0){ - // Do not emit comments which take a full line - const skipCommentLine = this.state.commenting && (this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0); - if(skipCommentLine){ - this.info.comment_lines++; - // Skip full comment line - }else { - // Activate records emition if above from_line - if(this.state.enabled === false && this.info.lines + (this.state.wasRowDelimiter === true ? 1: 0) >= from_line){ - this.state.enabled = true; - this.__resetField(); - this.__resetRecord(); - pos += recordDelimiterLength - 1; + if(this.state.field.length !== 0){ + // In relax_quotes mode, treat opening quote preceded by chrs as regular + if(relax_quotes === false){ + const err = this.__error( + new CsvError$1('INVALID_OPENING_QUOTE', [ + 'Invalid Opening Quote:', + `a quote is found inside a field at line ${this.info.lines}`, + ], this.options, this.__infoField(), { + field: this.state.field, + }) + ); + if(err !== undefined) return err; + } + }else { + this.state.quoting = true; + pos += quote.length - 1; continue; } - // Skip if line is empty and skip_empty_lines activated - if(skip_empty_lines === true && this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0){ - this.info.empty_lines++; - pos += recordDelimiterLength - 1; - continue; + } + } + if(this.state.quoting === false){ + const recordDelimiterLength = this.__isRecordDelimiter(chr, buf, pos); + if(recordDelimiterLength !== 0){ + // Do not emit comments which take a full line + const skipCommentLine = this.state.commenting && (this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0); + if(skipCommentLine){ + this.info.comment_lines++; + // Skip full comment line + }else { + // Activate records emition if above from_line + if(this.state.enabled === false && this.info.lines + (this.state.wasRowDelimiter === true ? 1: 0) >= from_line){ + this.state.enabled = true; + this.__resetField(); + this.__resetRecord(); + pos += recordDelimiterLength - 1; + continue; + } + // Skip if line is empty and skip_empty_lines activated + if(skip_empty_lines === true && this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0){ + this.info.empty_lines++; + pos += recordDelimiterLength - 1; + continue; + } + this.info.bytes = this.state.bufBytesStart + pos; + const errField = this.__onField(); + if(errField !== undefined) return errField; + this.info.bytes = this.state.bufBytesStart + pos + recordDelimiterLength; + const errRecord = this.__onRecord(push); + if(errRecord !== undefined) return errRecord; + if(to !== -1 && this.info.records >= to){ + this.state.stop = true; + close(); + return; + } } + this.state.commenting = false; + pos += recordDelimiterLength - 1; + continue; + } + if(this.state.commenting){ + continue; + } + const commentCount = comment === null ? 0 : this.__compareBytes(comment, buf, pos, chr); + if(commentCount !== 0){ + this.state.commenting = true; + continue; + } + const delimiterLength = this.__isDelimiter(buf, pos, chr); + if(delimiterLength !== 0){ this.info.bytes = this.state.bufBytesStart + pos; const errField = this.__onField(); if(errField !== undefined) return errField; - this.info.bytes = this.state.bufBytesStart + pos + recordDelimiterLength; - const errRecord = this.__onRecord(); - if(errRecord !== undefined) return errRecord; - if(to !== -1 && this.info.records >= to){ - this.state.stop = true; - this.push(null); - return; - } + pos += delimiterLength - 1; + continue; } - this.state.commenting = false; - pos += recordDelimiterLength - 1; - continue; - } - if(this.state.commenting){ - continue; - } - const commentCount = comment === null ? 0 : this.__compareBytes(comment, buf, pos, chr); - if(commentCount !== 0){ - this.state.commenting = true; - continue; } - const delimiterLength = this.__isDelimiter(buf, pos, chr); - if(delimiterLength !== 0){ - this.info.bytes = this.state.bufBytesStart + pos; - const errField = this.__onField(); - if(errField !== undefined) return errField; - pos += delimiterLength - 1; - continue; + } + if(this.state.commenting === false){ + if(max_record_size !== 0 && this.state.record_length + this.state.field.length > max_record_size){ + const err = this.__error( + new CsvError$1('CSV_MAX_RECORD_SIZE', [ + 'Max Record Size:', + 'record exceed the maximum number of tolerated bytes', + `of ${max_record_size}`, + `at line ${this.info.lines}`, + ], this.options, this.__infoField()) + ); + if(err !== undefined) return err; } } - } - if(this.state.commenting === false){ - if(max_record_size !== 0 && this.state.record_length + this.state.field.length > max_record_size){ + const lappend = ltrim === false || this.state.quoting === true || this.state.field.length !== 0 || !this.__isCharTrimable(chr); + // rtrim in non quoting is handle in __onField + const rappend = rtrim === false || this.state.wasQuoting === false; + if(lappend === true && rappend === true){ + this.state.field.append(chr); + }else if(rtrim === true && !this.__isCharTrimable(chr)){ const err = this.__error( - new CsvError$1('CSV_MAX_RECORD_SIZE', [ - 'Max Record Size:', - 'record exceed the maximum number of tolerated bytes', - `of ${max_record_size}`, + new CsvError$1('CSV_NON_TRIMABLE_CHAR_AFTER_CLOSING_QUOTE', [ + 'Invalid Closing Quote:', + 'found non trimable byte after quote', `at line ${this.info.lines}`, ], this.options, this.__infoField()) ); if(err !== undefined) return err; } } - const lappend = ltrim === false || this.state.quoting === true || this.state.field.length !== 0 || !this.__isCharTrimable(chr); - // rtrim in non quoting is handle in __onField - const rappend = rtrim === false || this.state.wasQuoting === false; - if(lappend === true && rappend === true){ - this.state.field.append(chr); - }else if(rtrim === true && !this.__isCharTrimable(chr)){ - const err = this.__error( - new CsvError$1('CSV_NON_TRIMABLE_CHAR_AFTER_CLOSING_QUOTE', [ - 'Invalid Closing Quote:', - 'found non trimable byte after quote', - `at line ${this.info.lines}`, - ], this.options, this.__infoField()) - ); - if(err !== undefined) return err; - } - } - if(end === true){ - // Ensure we are not ending in a quoting state - if(this.state.quoting === true){ - const err = this.__error( - new CsvError$1('CSV_QUOTE_NOT_CLOSED', [ - 'Quote Not Closed:', - `the parsing is finished with an opening quote at line ${this.info.lines}`, - ], this.options, this.__infoField()) - ); - if(err !== undefined) return err; + if(end === true){ + // Ensure we are not ending in a quoting state + if(this.state.quoting === true){ + const err = this.__error( + new CsvError$1('CSV_QUOTE_NOT_CLOSED', [ + 'Quote Not Closed:', + `the parsing is finished with an opening quote at line ${this.info.lines}`, + ], this.options, this.__infoField()) + ); + if(err !== undefined) return err; + }else { + // Skip last line if it has no characters + if(this.state.wasQuoting === true || this.state.record.length !== 0 || this.state.field.length !== 0){ + this.info.bytes = this.state.bufBytesStart + pos; + const errField = this.__onField(); + if(errField !== undefined) return errField; + const errRecord = this.__onRecord(push); + if(errRecord !== undefined) return errRecord; + }else if(this.state.wasRowDelimiter === true){ + this.info.empty_lines++; + }else if(this.state.commenting === true){ + this.info.comment_lines++; + } + } }else { - // Skip last line if it has no characters - if(this.state.wasQuoting === true || this.state.record.length !== 0 || this.state.field.length !== 0){ - this.info.bytes = this.state.bufBytesStart + pos; - const errField = this.__onField(); - if(errField !== undefined) return errField; - const errRecord = this.__onRecord(); - if(errRecord !== undefined) return errRecord; - }else if(this.state.wasRowDelimiter === true){ - this.info.empty_lines++; - }else if(this.state.commenting === true){ - this.info.comment_lines++; + this.state.bufBytesStart += pos; + this.state.previousBuf = buf.slice(pos); + } + if(this.state.wasRowDelimiter === true){ + this.info.lines++; + this.state.wasRowDelimiter = false; + } + }, + __onRecord: function(push){ + const {columns, group_columns_by_name, encoding, info, from, relax_column_count, relax_column_count_less, relax_column_count_more, raw, skip_records_with_empty_values} = this.options; + const {enabled, record} = this.state; + if(enabled === false){ + return this.__resetRecord(); + } + // Convert the first line into column names + const recordLength = record.length; + if(columns === true){ + if(skip_records_with_empty_values === true && isRecordEmpty(record)){ + this.__resetRecord(); + return; + } + return this.__firstLineToColumns(record); + } + if(columns === false && this.info.records === 0){ + this.state.expectedRecordLength = recordLength; + } + if(recordLength !== this.state.expectedRecordLength){ + const err = columns === false ? + new CsvError$1('CSV_RECORD_INCONSISTENT_FIELDS_LENGTH', [ + 'Invalid Record Length:', + `expect ${this.state.expectedRecordLength},`, + `got ${recordLength} on line ${this.info.lines}`, + ], this.options, this.__infoField(), { + record: record, + }) + : + new CsvError$1('CSV_RECORD_INCONSISTENT_COLUMNS', [ + 'Invalid Record Length:', + `columns length is ${columns.length},`, // rename columns + `got ${recordLength} on line ${this.info.lines}`, + ], this.options, this.__infoField(), { + record: record, + }); + if(relax_column_count === true || + (relax_column_count_less === true && recordLength < this.state.expectedRecordLength) || + (relax_column_count_more === true && recordLength > this.state.expectedRecordLength)){ + this.info.invalid_field_length++; + this.state.error = err; + // Error is undefined with skip_records_with_error + }else { + const finalErr = this.__error(err); + if(finalErr) return finalErr; } } - }else { - this.state.bufBytesStart += pos; - this.state.previousBuf = buf.slice(pos); - } - if(this.state.wasRowDelimiter === true){ - this.info.lines++; - this.state.wasRowDelimiter = false; - } - } - __onRecord(){ - const {columns, group_columns_by_name, encoding, info, from, relax_column_count, relax_column_count_less, relax_column_count_more, raw, skip_records_with_empty_values} = this.options; - const {enabled, record} = this.state; - if(enabled === false){ - return this.__resetRecord(); - } - // Convert the first line into column names - const recordLength = record.length; - if(columns === true){ if(skip_records_with_empty_values === true && isRecordEmpty(record)){ this.__resetRecord(); return; } - return this.__firstLineToColumns(record); - } - if(columns === false && this.info.records === 0){ - this.state.expectedRecordLength = recordLength; - } - if(recordLength !== this.state.expectedRecordLength){ - const err = columns === false ? - new CsvError$1('CSV_RECORD_INCONSISTENT_FIELDS_LENGTH', [ - 'Invalid Record Length:', - `expect ${this.state.expectedRecordLength},`, - `got ${recordLength} on line ${this.info.lines}`, - ], this.options, this.__infoField(), { - record: record, - }) - : - new CsvError$1('CSV_RECORD_INCONSISTENT_COLUMNS', [ - 'Invalid Record Length:', - `columns length is ${columns.length},`, // rename columns - `got ${recordLength} on line ${this.info.lines}`, - ], this.options, this.__infoField(), { - record: record, - }); - if(relax_column_count === true || - (relax_column_count_less === true && recordLength < this.state.expectedRecordLength) || - (relax_column_count_more === true && recordLength > this.state.expectedRecordLength)){ - this.info.invalid_field_length++; - this.state.error = err; - // Error is undefined with skip_records_with_error - }else { - const finalErr = this.__error(err); - if(finalErr) return finalErr; + if(this.state.recordHasError === true){ + this.__resetRecord(); + this.state.recordHasError = false; + return; } - } - if(skip_records_with_empty_values === true && isRecordEmpty(record)){ - this.__resetRecord(); - return; - } - if(this.state.recordHasError === true){ - this.__resetRecord(); - this.state.recordHasError = false; - return; - } - this.info.records++; - if(from === 1 || this.info.records >= from){ - const {objname} = this.options; - // With columns, records are object - if(columns !== false){ - const obj = {}; - // Transform record array to an object - for(let i = 0, l = record.length; i < l; i++){ - if(columns[i] === undefined || columns[i].disabled) continue; - // Turn duplicate columns into an array - if (group_columns_by_name === true && obj[columns[i].name] !== undefined) { - if (Array.isArray(obj[columns[i].name])) { - obj[columns[i].name] = obj[columns[i].name].concat(record[i]); + this.info.records++; + if(from === 1 || this.info.records >= from){ + const {objname} = this.options; + // With columns, records are object + if(columns !== false){ + const obj = {}; + // Transform record array to an object + for(let i = 0, l = record.length; i < l; i++){ + if(columns[i] === undefined || columns[i].disabled) continue; + // Turn duplicate columns into an array + if (group_columns_by_name === true && obj[columns[i].name] !== undefined) { + if (Array.isArray(obj[columns[i].name])) { + obj[columns[i].name] = obj[columns[i].name].concat(record[i]); + } else { + obj[columns[i].name] = [obj[columns[i].name], record[i]]; + } } else { - obj[columns[i].name] = [obj[columns[i].name], record[i]]; + obj[columns[i].name] = record[i]; } - } else { - obj[columns[i].name] = record[i]; } - } - // Without objname (default) - if(raw === true || info === true){ - const extRecord = Object.assign( - {record: obj}, - (raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}), - (info === true ? {info: this.__infoRecord()}: {}) - ); - const err = this.__push( - objname === undefined ? extRecord : [obj[objname], extRecord] - ); - if(err){ - return err; - } - }else { - const err = this.__push( - objname === undefined ? obj : [obj[objname], obj] - ); - if(err){ - return err; - } - } - // Without columns, records are array - }else { - if(raw === true || info === true){ - const extRecord = Object.assign( - {record: record}, - raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}, - info === true ? {info: this.__infoRecord()}: {} - ); - const err = this.__push( - objname === undefined ? extRecord : [record[objname], extRecord] - ); - if(err){ - return err; + // Without objname (default) + if(raw === true || info === true){ + const extRecord = Object.assign( + {record: obj}, + (raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}), + (info === true ? {info: this.__infoRecord()}: {}) + ); + const err = this.__push( + objname === undefined ? extRecord : [obj[objname], extRecord] + , push); + if(err){ + return err; + } + }else { + const err = this.__push( + objname === undefined ? obj : [obj[objname], obj] + , push); + if(err){ + return err; + } } + // Without columns, records are array }else { - const err = this.__push( - objname === undefined ? record : [record[objname], record] - ); - if(err){ - return err; + if(raw === true || info === true){ + const extRecord = Object.assign( + {record: record}, + raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}, + info === true ? {info: this.__infoRecord()}: {} + ); + const err = this.__push( + objname === undefined ? extRecord : [record[objname], extRecord] + , push); + if(err){ + return err; + } + }else { + const err = this.__push( + objname === undefined ? record : [record[objname], record] + , push); + if(err){ + return err; + } } } } - } - this.__resetRecord(); - } - __firstLineToColumns(record){ - const {firstLineToHeaders} = this.state; - try{ - const headers = firstLineToHeaders === undefined ? record : firstLineToHeaders.call(null, record); - if(!Array.isArray(headers)){ - return this.__error( - new CsvError$1('CSV_INVALID_COLUMN_MAPPING', [ - 'Invalid Column Mapping:', - 'expect an array from column function,', - `got ${JSON.stringify(headers)}` - ], this.options, this.__infoField(), { - headers: headers, - }) - ); - } - const normalizedHeaders = normalizeColumnsArray(headers); - this.state.expectedRecordLength = normalizedHeaders.length; - this.options.columns = normalizedHeaders; this.__resetRecord(); - return; - }catch(err){ - return err; - } - } - __resetRecord(){ - if(this.options.raw === true){ - this.state.rawBuffer.reset(); - } - this.state.error = undefined; - this.state.record = []; - this.state.record_length = 0; - } - __onField(){ - const {cast, encoding, rtrim, max_record_size} = this.options; - const {enabled, wasQuoting} = this.state; - // Short circuit for the from_line options - if(enabled === false){ - return this.__resetField(); - } - let field = this.state.field.toString(encoding); - if(rtrim === true && wasQuoting === false){ - field = field.trimRight(); - } - if(cast === true){ - const [err, f] = this.__cast(field); - if(err !== undefined) return err; - field = f; - } - this.state.record.push(field); - // Increment record length if record size must not exceed a limit - if(max_record_size !== 0 && typeof field === 'string'){ - this.state.record_length += field.length; - } - this.__resetField(); - } - __resetField(){ - this.state.field.reset(); - this.state.wasQuoting = false; - } - __push(record){ - const {on_record} = this.options; - if(on_record !== undefined){ - const info = this.__infoRecord(); + }, + __firstLineToColumns: function(record){ + const {firstLineToHeaders} = this.state; try{ - record = on_record.call(null, record, info); + const headers = firstLineToHeaders === undefined ? record : firstLineToHeaders.call(null, record); + if(!Array.isArray(headers)){ + return this.__error( + new CsvError$1('CSV_INVALID_COLUMN_MAPPING', [ + 'Invalid Column Mapping:', + 'expect an array from column function,', + `got ${JSON.stringify(headers)}` + ], this.options, this.__infoField(), { + headers: headers, + }) + ); + } + const normalizedHeaders = normalize_columns_array(headers); + this.state.expectedRecordLength = normalizedHeaders.length; + this.options.columns = normalizedHeaders; + this.__resetRecord(); + return; }catch(err){ return err; } - if(record === undefined || record === null){ return; } - } - this.push(record); - } - // Return a tuple with the error and the casted value - __cast(field){ - const {columns, relax_column_count} = this.options; - const isColumns = Array.isArray(columns); - // Dont loose time calling cast - // because the final record is an object - // and this field can't be associated to a key present in columns - if(isColumns === true && relax_column_count && this.options.columns.length <= this.state.record.length){ - return [undefined, undefined]; - } - if(this.state.castField !== null){ - try{ + }, + __resetRecord: function(){ + if(this.options.raw === true){ + this.state.rawBuffer.reset(); + } + this.state.error = undefined; + this.state.record = []; + this.state.record_length = 0; + }, + __onField: function(){ + const {cast, encoding, rtrim, max_record_size} = this.options; + const {enabled, wasQuoting} = this.state; + // Short circuit for the from_line options + if(enabled === false){ + return this.__resetField(); + } + let field = this.state.field.toString(encoding); + if(rtrim === true && wasQuoting === false){ + field = field.trimRight(); + } + if(cast === true){ + const [err, f] = this.__cast(field); + if(err !== undefined) return err; + field = f; + } + this.state.record.push(field); + // Increment record length if record size must not exceed a limit + if(max_record_size !== 0 && typeof field === 'string'){ + this.state.record_length += field.length; + } + this.__resetField(); + }, + __resetField: function(){ + this.state.field.reset(); + this.state.wasQuoting = false; + }, + __push: function(record, push){ + const {on_record} = this.options; + if(on_record !== undefined){ + const info = this.__infoRecord(); + try{ + record = on_record.call(null, record, info); + }catch(err){ + return err; + } + if(record === undefined || record === null){ return; } + } + push(record); + }, + // Return a tuple with the error and the casted value + __cast: function(field){ + const {columns, relax_column_count} = this.options; + const isColumns = Array.isArray(columns); + // Dont loose time calling cast + // because the final record is an object + // and this field can't be associated to a key present in columns + if(isColumns === true && relax_column_count && this.options.columns.length <= this.state.record.length){ + return [undefined, undefined]; + } + if(this.state.castField !== null){ + try{ + const info = this.__infoField(); + return [undefined, this.state.castField.call(null, field, info)]; + }catch(err){ + return [err]; + } + } + if(this.__isFloat(field)){ + return [undefined, parseFloat(field)]; + }else if(this.options.cast_date !== false){ const info = this.__infoField(); - return [undefined, this.state.castField.call(null, field, info)]; - }catch(err){ - return [err]; + return [undefined, this.options.cast_date.call(null, field, info)]; } - } - if(this.__isFloat(field)){ - return [undefined, parseFloat(field)]; - }else if(this.options.cast_date !== false){ - const info = this.__infoField(); - return [undefined, this.options.cast_date.call(null, field, info)]; - } - return [undefined, field]; - } - // Helper to test if a character is a space or a line delimiter - __isCharTrimable(chr){ - return chr === space || chr === tab || chr === cr || chr === nl || chr === np; - } - // Keep it in case we implement the `cast_int` option - // __isInt(value){ - // // return Number.isInteger(parseInt(value)) - // // return !isNaN( parseInt( obj ) ); - // return /^(\-|\+)?[1-9][0-9]*$/.test(value) - // } - __isFloat(value){ - return (value - parseFloat(value) + 1) >= 0; // Borrowed from jquery - } - __compareBytes(sourceBuf, targetBuf, targetPos, firstByte){ - if(sourceBuf[0] !== firstByte) return 0; - const sourceLength = sourceBuf.length; - for(let i = 1; i < sourceLength; i++){ - if(sourceBuf[i] !== targetBuf[targetPos+i]) return 0; - } - return sourceLength; - } - __needMoreData(i, bufLen, end){ - if(end) return false; - const {quote} = this.options; - const {quoting, needMoreDataSize, recordDelimiterMaxLength} = this.state; - const numOfCharLeft = bufLen - i - 1; - const requiredLength = Math.max( - needMoreDataSize, - // Skip if the remaining buffer smaller than record delimiter - recordDelimiterMaxLength, - // Skip if the remaining buffer can be record delimiter following the closing quote - // 1 is for quote.length - quoting ? (quote.length + recordDelimiterMaxLength) : 0, - ); - return numOfCharLeft < requiredLength; - } - __isDelimiter(buf, pos, chr){ - const {delimiter, ignore_last_delimiters} = this.options; - if(ignore_last_delimiters === true && this.state.record.length === this.options.columns.length - 1){ - return 0; - }else if(ignore_last_delimiters !== false && typeof ignore_last_delimiters === 'number' && this.state.record.length === ignore_last_delimiters - 1){ - return 0; - } - loop1: for(let i = 0; i < delimiter.length; i++){ - const del = delimiter[i]; - if(del[0] === chr){ - for(let j = 1; j < del.length; j++){ - if(del[j] !== buf[pos+j]) continue loop1; + return [undefined, field]; + }, + // Helper to test if a character is a space or a line delimiter + __isCharTrimable: function(chr){ + return chr === space || chr === tab || chr === cr || chr === nl || chr === np; + }, + // Keep it in case we implement the `cast_int` option + // __isInt(value){ + // // return Number.isInteger(parseInt(value)) + // // return !isNaN( parseInt( obj ) ); + // return /^(\-|\+)?[1-9][0-9]*$/.test(value) + // } + __isFloat: function(value){ + return (value - parseFloat(value) + 1) >= 0; // Borrowed from jquery + }, + __compareBytes: function(sourceBuf, targetBuf, targetPos, firstByte){ + if(sourceBuf[0] !== firstByte) return 0; + const sourceLength = sourceBuf.length; + for(let i = 1; i < sourceLength; i++){ + if(sourceBuf[i] !== targetBuf[targetPos+i]) return 0; + } + return sourceLength; + }, + __isDelimiter: function(buf, pos, chr){ + const {delimiter, ignore_last_delimiters} = this.options; + if(ignore_last_delimiters === true && this.state.record.length === this.options.columns.length - 1){ + return 0; + }else if(ignore_last_delimiters !== false && typeof ignore_last_delimiters === 'number' && this.state.record.length === ignore_last_delimiters - 1){ + return 0; + } + loop1: for(let i = 0; i < delimiter.length; i++){ + const del = delimiter[i]; + if(del[0] === chr){ + for(let j = 1; j < del.length; j++){ + if(del[j] !== buf[pos+j]) continue loop1; + } + return del.length; } - return del.length; } - } - return 0; - } - __isRecordDelimiter(chr, buf, pos){ - const {record_delimiter} = this.options; - const recordDelimiterLength = record_delimiter.length; - loop1: for(let i = 0; i < recordDelimiterLength; i++){ - const rd = record_delimiter[i]; - const rdLength = rd.length; - if(rd[0] !== chr){ - continue; + return 0; + }, + __isRecordDelimiter: function(chr, buf, pos){ + const {record_delimiter} = this.options; + const recordDelimiterLength = record_delimiter.length; + loop1: for(let i = 0; i < recordDelimiterLength; i++){ + const rd = record_delimiter[i]; + const rdLength = rd.length; + if(rd[0] !== chr){ + continue; + } + for(let j = 1; j < rdLength; j++){ + if(rd[j] !== buf[pos+j]){ + continue loop1; + } + } + return rd.length; } - for(let j = 1; j < rdLength; j++){ - if(rd[j] !== buf[pos+j]){ - continue loop1; + return 0; + }, + __isEscape: function(buf, pos, chr){ + const {escape} = this.options; + if(escape === null) return false; + const l = escape.length; + if(escape[0] === chr){ + for(let i = 0; i < l; i++){ + if(escape[i] !== buf[pos+i]){ + return false; + } } + return true; } - return rd.length; - } - return 0; - } - __isEscape(buf, pos, chr){ - const {escape} = this.options; - if(escape === null) return false; - const l = escape.length; - if(escape[0] === chr){ + return false; + }, + __isQuote: function(buf, pos){ + const {quote} = this.options; + if(quote === null) return false; + const l = quote.length; for(let i = 0; i < l; i++){ - if(escape[i] !== buf[pos+i]){ + if(quote[i] !== buf[pos+i]){ return false; } } return true; - } - return false; - } - __isQuote(buf, pos){ - const {quote} = this.options; - if(quote === null) return false; - const l = quote.length; - for(let i = 0; i < l; i++){ - if(quote[i] !== buf[pos+i]){ - return false; - } - } - return true; - } - __autoDiscoverRecordDelimiter(buf, pos){ - const {encoding} = this.options; - const chr = buf[pos]; - if(chr === cr){ - if(buf[pos+1] === nl){ - this.options.record_delimiter.push(Buffer.from('\r\n', encoding)); - this.state.recordDelimiterMaxLength = 2; - return 2; - }else { - this.options.record_delimiter.push(Buffer.from('\r', encoding)); + }, + __autoDiscoverRecordDelimiter: function(buf, pos){ + const {encoding} = this.options; + const chr = buf[pos]; + if(chr === cr){ + if(buf[pos+1] === nl){ + this.options.record_delimiter.push(Buffer.from('\r\n', encoding)); + this.state.recordDelimiterMaxLength = 2; + return 2; + }else { + this.options.record_delimiter.push(Buffer.from('\r', encoding)); + this.state.recordDelimiterMaxLength = 1; + return 1; + } + }else if(chr === nl){ + this.options.record_delimiter.push(Buffer.from('\n', encoding)); this.state.recordDelimiterMaxLength = 1; return 1; } - }else if(chr === nl){ - this.options.record_delimiter.push(Buffer.from('\n', encoding)); - this.state.recordDelimiterMaxLength = 1; - return 1; - } - return 0; - } - __error(msg){ - const {encoding, raw, skip_records_with_error} = this.options; - const err = typeof msg === 'string' ? new Error(msg) : msg; - if(skip_records_with_error){ - this.state.recordHasError = true; - this.emit('skip', err, raw ? this.state.rawBuffer.toString(encoding) : undefined); - return undefined; - }else { - return err; + return 0; + }, + __error: function(msg){ + const {encoding, raw, skip_records_with_error} = this.options; + const err = typeof msg === 'string' ? new Error(msg) : msg; + if(skip_records_with_error){ + this.state.recordHasError = true; + if(this.options.on_skip !== undefined){ + this.options.on_skip(err, raw ? this.state.rawBuffer.toString(encoding) : undefined); + } + // this.emit('skip', err, raw ? this.state.rawBuffer.toString(encoding) : undefined); + return undefined; + }else { + return err; + } + }, + __infoDataSet: function(){ + return { + ...this.info, + columns: this.options.columns + }; + }, + __infoRecord: function(){ + const {columns, raw, encoding} = this.options; + return { + ...this.__infoDataSet(), + error: this.state.error, + header: columns === true, + index: this.state.record.length, + raw: raw ? this.state.rawBuffer.toString(encoding) : undefined + }; + }, + __infoField: function(){ + const {columns} = this.options; + const isColumns = Array.isArray(columns); + return { + ...this.__infoRecord(), + column: isColumns === true ? + (columns.length > this.state.record.length ? + columns[this.state.record.length].name : + null + ) : + this.state.record.length, + quoting: this.state.wasQuoting, + }; } - } - __infoDataSet(){ - return { - ...this.info, - columns: this.options.columns - }; - } - __infoRecord(){ - const {columns, raw, encoding} = this.options; - return { - ...this.__infoDataSet(), - error: this.state.error, - header: columns === true, - index: this.state.record.length, - raw: raw ? this.state.rawBuffer.toString(encoding) : undefined - }; - } - __infoField(){ - const {columns} = this.options; - const isColumns = Array.isArray(columns); - return { - ...this.__infoRecord(), - column: isColumns === true ? - (columns.length > this.state.record.length ? - columns[this.state.record.length].name : - null - ) : - this.state.record.length, - quoting: this.state.wasQuoting, - }; - } - } + }; + }; - const parse = function(data, options={}){ + const parse = function(data, opts={}){ if(typeof data === 'string'){ data = Buffer.from(data); } - const records = options && options.objname ? {} : []; - const parser = new Parser(options); - parser.push = function(record){ - if(record === null){ - return; - } - if(options.objname === undefined) + const records = opts && opts.objname ? {} : []; + const parser = transform$1(opts); + const push = (record) => { + if(parser.options.objname === undefined) records.push(record); else { records[record[0]] = record[1]; } }; - const err1 = parser.__parse(data, false); + const close = () => {}; + const err1 = parser.parse(data, false, push, close); if(err1 !== undefined) throw err1; - const err2 = parser.__parse(undefined, true); + const err2 = parser.parse(undefined, true, push, close); if(err2 !== undefined) throw err2; return records; }; - const bom_utf8 = Buffer.from([239, 187, 191]); - - class CsvError extends Error { - constructor(code, message, ...contexts) { - if(Array.isArray(message)) message = message.join(' '); - super(message); - if(Error.captureStackTrace !== undefined){ - Error.captureStackTrace(this, CsvError); - } - this.code = code; - for(const context of contexts){ - for(const key in context){ - const value = context[key]; - this[key] = isBuffer$1(value) ? value.toString() : value == null ? value : JSON.parse(JSON.stringify(value)); - } - } - } - } - - const isObject = function(obj){ - return typeof obj === 'object' && obj !== null && ! Array.isArray(obj); - }; - - const underscore = function(str){ - return str.replace(/([A-Z])/g, function(_, match){ - return '_' + match.toLowerCase(); - }); - }; - // Lodash implementation of `get` const charCodeOfDot = '.'.charCodeAt(0); @@ -6653,455 +6645,473 @@ var csv_sync = (function (exports) { return (index && index === length) ? object : undefined; }; - class Stringifier extends Transform { - constructor(opts = {}){ - super({...{writableObjectMode: true}, ...opts}); - const options = {}; - let err; - // Merge with user options - for(const opt in opts){ - options[underscore(opt)] = opts[opt]; - } - if((err = this.normalize(options)) !== undefined) throw err; - switch(options.record_delimiter){ - case 'auto': - options.record_delimiter = null; - break; - case 'unix': - options.record_delimiter = "\n"; - break; - case 'mac': - options.record_delimiter = "\r"; - break; - case 'windows': - options.record_delimiter = "\r\n"; - break; - case 'ascii': - options.record_delimiter = "\u001e"; - break; - case 'unicode': - options.record_delimiter = "\u2028"; - break; - } - // Expose options - this.options = options; - // Internal state - this.state = { - stop: false - }; - // Information - this.info = { - records: 0 - }; + const is_object = function(obj){ + return typeof obj === 'object' && obj !== null && ! Array.isArray(obj); + }; + + const normalize_columns = function(columns){ + if(columns === undefined || columns === null){ + return [undefined, undefined]; } - normalize(options){ - // Normalize option `bom` - if(options.bom === undefined || options.bom === null || options.bom === false){ - options.bom = false; - }else if(options.bom !== true){ - return new CsvError('CSV_OPTION_BOOLEAN_INVALID_TYPE', [ - 'option `bom` is optional and must be a boolean value,', - `got ${JSON.stringify(options.bom)}` - ]); - } - // Normalize option `delimiter` - if(options.delimiter === undefined || options.delimiter === null){ - options.delimiter = ','; - }else if(isBuffer$1(options.delimiter)){ - options.delimiter = options.delimiter.toString(); - }else if(typeof options.delimiter !== 'string'){ - return new CsvError('CSV_OPTION_DELIMITER_INVALID_TYPE', [ - 'option `delimiter` must be a buffer or a string,', - `got ${JSON.stringify(options.delimiter)}` - ]); - } - // Normalize option `quote` - if(options.quote === undefined || options.quote === null){ - options.quote = '"'; - }else if(options.quote === true){ - options.quote = '"'; - }else if(options.quote === false){ - options.quote = ''; - }else if (isBuffer$1(options.quote)){ - options.quote = options.quote.toString(); - }else if(typeof options.quote !== 'string'){ - return new CsvError('CSV_OPTION_QUOTE_INVALID_TYPE', [ - 'option `quote` must be a boolean, a buffer or a string,', - `got ${JSON.stringify(options.quote)}` - ]); + if(typeof columns !== 'object'){ + return [Error('Invalid option "columns": expect an array or an object')]; + } + if(!Array.isArray(columns)){ + const newcolumns = []; + for(const k in columns){ + newcolumns.push({ + key: k, + header: columns[k] + }); } - // Normalize option `quoted` - if(options.quoted === undefined || options.quoted === null){ - options.quoted = false; - } - // Normalize option `quoted_empty` - if(options.quoted_empty === undefined || options.quoted_empty === null){ - options.quoted_empty = undefined; - } - // Normalize option `quoted_match` - if(options.quoted_match === undefined || options.quoted_match === null || options.quoted_match === false){ - options.quoted_match = null; - }else if(!Array.isArray(options.quoted_match)){ - options.quoted_match = [options.quoted_match]; - } - if(options.quoted_match){ - for(const quoted_match of options.quoted_match){ - const isString = typeof quoted_match === 'string'; - const isRegExp = quoted_match instanceof RegExp; - if(!isString && !isRegExp){ - return Error(`Invalid Option: quoted_match must be a string or a regex, got ${JSON.stringify(quoted_match)}`); + columns = newcolumns; + }else { + const newcolumns = []; + for(const column of columns){ + if(typeof column === 'string'){ + newcolumns.push({ + key: column, + header: column + }); + }else if(typeof column === 'object' && column !== null && !Array.isArray(column)){ + if(!column.key){ + return [Error('Invalid column definition: property "key" is required')]; } + if(column.header === undefined){ + column.header = column.key; + } + newcolumns.push(column); + }else { + return [Error('Invalid column definition: expect a string or an object')]; } } - // Normalize option `quoted_string` - if(options.quoted_string === undefined || options.quoted_string === null){ - options.quoted_string = false; - } - // Normalize option `eof` - if(options.eof === undefined || options.eof === null){ - options.eof = true; - } - // Normalize option `escape` - if(options.escape === undefined || options.escape === null){ - options.escape = '"'; - }else if(isBuffer$1(options.escape)){ - options.escape = options.escape.toString(); - }else if(typeof options.escape !== 'string'){ - return Error(`Invalid Option: escape must be a buffer or a string, got ${JSON.stringify(options.escape)}`); - } - if (options.escape.length > 1){ - return Error(`Invalid Option: escape must be one character, got ${options.escape.length} characters`); - } - // Normalize option `header` - if(options.header === undefined || options.header === null){ - options.header = false; - } - // Normalize option `columns` - const [err, columns] = this.normalize_columns(options.columns); - if(err) return err; - options.columns = columns; - // Normalize option `quoted` - if(options.quoted === undefined || options.quoted === null){ - options.quoted = false; - } - // Normalize option `cast` - if(options.cast === undefined || options.cast === null){ - options.cast = {}; - } - // Normalize option cast.bigint - if(options.cast.bigint === undefined || options.cast.bigint === null){ - // Cast boolean to string by default - options.cast.bigint = value => '' + value; - } - // Normalize option cast.boolean - if(options.cast.boolean === undefined || options.cast.boolean === null){ - // Cast boolean to string by default - options.cast.boolean = value => value ? '1' : ''; - } - // Normalize option cast.date - if(options.cast.date === undefined || options.cast.date === null){ - // Cast date to timestamp string by default - options.cast.date = value => '' + value.getTime(); - } - // Normalize option cast.number - if(options.cast.number === undefined || options.cast.number === null){ - // Cast number to string using native casting by default - options.cast.number = value => '' + value; - } - // Normalize option cast.object - if(options.cast.object === undefined || options.cast.object === null){ - // Stringify object as JSON by default - options.cast.object = value => JSON.stringify(value); - } - // Normalize option cast.string - if(options.cast.string === undefined || options.cast.string === null){ - // Leave string untouched - options.cast.string = function(value){return value;}; - } - // Normalize option `record_delimiter` - if(options.record_delimiter === undefined || options.record_delimiter === null){ - options.record_delimiter = '\n'; - }else if(isBuffer$1(options.record_delimiter)){ - options.record_delimiter = options.record_delimiter.toString(); - }else if(typeof options.record_delimiter !== 'string'){ - return Error(`Invalid Option: record_delimiter must be a buffer or a string, got ${JSON.stringify(options.record_delimiter)}`); - } - } - _transform(chunk, encoding, callback){ - if(this.state.stop === true){ - return; - } - const err = this.__transform(chunk); - if(err !== undefined){ - this.state.stop = true; - } - callback(err); + columns = newcolumns; } - _flush(callback){ - if(this.state.stop === true){ - // Note, Node.js 12 call flush even after an error, we must prevent - // `callback` from being called in flush without any error. - return; + return [undefined, columns]; + }; + + class CsvError extends Error { + constructor(code, message, ...contexts) { + if(Array.isArray(message)) message = message.join(' '); + super(message); + if(Error.captureStackTrace !== undefined){ + Error.captureStackTrace(this, CsvError); } - if(this.info.records === 0){ - this.bom(); - const err = this.headers(); - if(err) callback(err); + this.code = code; + for(const context of contexts){ + for(const key in context){ + const value = context[key]; + this[key] = isBuffer$1(value) ? value.toString() : value == null ? value : JSON.parse(JSON.stringify(value)); + } } - callback(); } - __transform(chunk){ - // Chunk validation - if(!Array.isArray(chunk) && typeof chunk !== 'object'){ - return Error(`Invalid Record: expect an array or an object, got ${JSON.stringify(chunk)}`); - } - // Detect columns from the first record - if(this.info.records === 0){ - if(Array.isArray(chunk)){ - if(this.options.header === true && this.options.columns === undefined){ - return Error('Undiscoverable Columns: header option requires column option or object records'); - } - }else if(this.options.columns === undefined){ - const [err, columns] = this.normalize_columns(Object.keys(chunk)); - if(err) return; - this.options.columns = columns; + } + + const underscore = function(str){ + return str.replace(/([A-Z])/g, function(_, match){ + return '_' + match.toLowerCase(); + }); + }; + + const normalize_options = function(opts) { + const options = {}; + // Merge with user options + for(const opt in opts){ + options[underscore(opt)] = opts[opt]; + } + // Normalize option `bom` + if(options.bom === undefined || options.bom === null || options.bom === false){ + options.bom = false; + }else if(options.bom !== true){ + return [new CsvError('CSV_OPTION_BOOLEAN_INVALID_TYPE', [ + 'option `bom` is optional and must be a boolean value,', + `got ${JSON.stringify(options.bom)}` + ])]; + } + // Normalize option `delimiter` + if(options.delimiter === undefined || options.delimiter === null){ + options.delimiter = ','; + }else if(isBuffer$1(options.delimiter)){ + options.delimiter = options.delimiter.toString(); + }else if(typeof options.delimiter !== 'string'){ + return [new CsvError('CSV_OPTION_DELIMITER_INVALID_TYPE', [ + 'option `delimiter` must be a buffer or a string,', + `got ${JSON.stringify(options.delimiter)}` + ])]; + } + // Normalize option `quote` + if(options.quote === undefined || options.quote === null){ + options.quote = '"'; + }else if(options.quote === true){ + options.quote = '"'; + }else if(options.quote === false){ + options.quote = ''; + }else if (isBuffer$1(options.quote)){ + options.quote = options.quote.toString(); + }else if(typeof options.quote !== 'string'){ + return [new CsvError('CSV_OPTION_QUOTE_INVALID_TYPE', [ + 'option `quote` must be a boolean, a buffer or a string,', + `got ${JSON.stringify(options.quote)}` + ])]; + } + // Normalize option `quoted` + if(options.quoted === undefined || options.quoted === null){ + options.quoted = false; + } + // Normalize option `quoted_empty` + if(options.quoted_empty === undefined || options.quoted_empty === null){ + options.quoted_empty = undefined; + } + // Normalize option `quoted_match` + if(options.quoted_match === undefined || options.quoted_match === null || options.quoted_match === false){ + options.quoted_match = null; + }else if(!Array.isArray(options.quoted_match)){ + options.quoted_match = [options.quoted_match]; + } + if(options.quoted_match){ + for(const quoted_match of options.quoted_match){ + const isString = typeof quoted_match === 'string'; + const isRegExp = quoted_match instanceof RegExp; + if(!isString && !isRegExp){ + return [Error(`Invalid Option: quoted_match must be a string or a regex, got ${JSON.stringify(quoted_match)}`)]; } } - // Emit the header - if(this.info.records === 0){ - this.bom(); - const err = this.headers(); - if(err) return err; - } - // Emit and stringify the record if an object or an array - try{ - this.emit('record', chunk, this.info.records); - }catch(err){ - return err; - } - // Convert the record into a string - let err, chunk_string; - if(this.options.eof){ - [err, chunk_string] = this.stringify(chunk); - if(err) return err; - if(chunk_string === undefined){ - return; - }else { - chunk_string = chunk_string + this.options.record_delimiter; + } + // Normalize option `quoted_string` + if(options.quoted_string === undefined || options.quoted_string === null){ + options.quoted_string = false; + } + // Normalize option `eof` + if(options.eof === undefined || options.eof === null){ + options.eof = true; + } + // Normalize option `escape` + if(options.escape === undefined || options.escape === null){ + options.escape = '"'; + }else if(isBuffer$1(options.escape)){ + options.escape = options.escape.toString(); + }else if(typeof options.escape !== 'string'){ + return [Error(`Invalid Option: escape must be a buffer or a string, got ${JSON.stringify(options.escape)}`)]; + } + if (options.escape.length > 1){ + return [Error(`Invalid Option: escape must be one character, got ${options.escape.length} characters`)]; + } + // Normalize option `header` + if(options.header === undefined || options.header === null){ + options.header = false; + } + // Normalize option `columns` + const [errColumns, columns] = normalize_columns(options.columns); + if(errColumns !== undefined) return [errColumns]; + options.columns = columns; + // Normalize option `quoted` + if(options.quoted === undefined || options.quoted === null){ + options.quoted = false; + } + // Normalize option `cast` + if(options.cast === undefined || options.cast === null){ + options.cast = {}; + } + // Normalize option cast.bigint + if(options.cast.bigint === undefined || options.cast.bigint === null){ + // Cast boolean to string by default + options.cast.bigint = value => '' + value; + } + // Normalize option cast.boolean + if(options.cast.boolean === undefined || options.cast.boolean === null){ + // Cast boolean to string by default + options.cast.boolean = value => value ? '1' : ''; + } + // Normalize option cast.date + if(options.cast.date === undefined || options.cast.date === null){ + // Cast date to timestamp string by default + options.cast.date = value => '' + value.getTime(); + } + // Normalize option cast.number + if(options.cast.number === undefined || options.cast.number === null){ + // Cast number to string using native casting by default + options.cast.number = value => '' + value; + } + // Normalize option cast.object + if(options.cast.object === undefined || options.cast.object === null){ + // Stringify object as JSON by default + options.cast.object = value => JSON.stringify(value); + } + // Normalize option cast.string + if(options.cast.string === undefined || options.cast.string === null){ + // Leave string untouched + options.cast.string = function(value){return value;}; + } + // Normalize option `on_record` + if(options.on_record !== undefined && typeof options.on_record !== 'function'){ + return [Error(`Invalid Option: "on_record" must be a function.`)]; + } + // Normalize option `record_delimiter` + if(options.record_delimiter === undefined || options.record_delimiter === null){ + options.record_delimiter = '\n'; + }else if(isBuffer$1(options.record_delimiter)){ + options.record_delimiter = options.record_delimiter.toString(); + }else if(typeof options.record_delimiter !== 'string'){ + return [Error(`Invalid Option: record_delimiter must be a buffer or a string, got ${JSON.stringify(options.record_delimiter)}`)]; + } + switch(options.record_delimiter){ + case 'auto': + options.record_delimiter = null; + break; + case 'unix': + options.record_delimiter = "\n"; + break; + case 'mac': + options.record_delimiter = "\r"; + break; + case 'windows': + options.record_delimiter = "\r\n"; + break; + case 'ascii': + options.record_delimiter = "\u001e"; + break; + case 'unicode': + options.record_delimiter = "\u2028"; + break; + } + return [undefined, options]; + }; + + const bom_utf8 = Buffer.from([239, 187, 191]); + + const stringifier = function(options, state, info){ + return { + options: options, + state: state, + info: info, + __transform: function(chunk, push){ + // Chunk validation + if(!Array.isArray(chunk) && typeof chunk !== 'object'){ + return Error(`Invalid Record: expect an array or an object, got ${JSON.stringify(chunk)}`); } - }else { - [err, chunk_string] = this.stringify(chunk); - if(err) return err; - if(chunk_string === undefined){ - return; - }else { - if(this.options.header || this.info.records){ - chunk_string = this.options.record_delimiter + chunk_string; + // Detect columns from the first record + if(this.info.records === 0){ + if(Array.isArray(chunk)){ + if(this.options.header === true && this.options.columns === undefined){ + return Error('Undiscoverable Columns: header option requires column option or object records'); + } + }else if(this.options.columns === undefined){ + const [err, columns] = normalize_columns(Object.keys(chunk)); + if(err) return; + this.options.columns = columns; } } - } - // Emit the csv - this.info.records++; - this.push(chunk_string); - } - stringify(chunk, chunkIsHeader=false){ - if(typeof chunk !== 'object'){ - return [undefined, chunk]; - } - const {columns} = this.options; - const record = []; - // Record is an array - if(Array.isArray(chunk)){ - // We are getting an array but the user has specified output columns. In - // this case, we respect the columns indexes - if(columns){ - chunk.splice(columns.length); + // Emit the header + if(this.info.records === 0){ + this.bom(push); + const err = this.headers(push); + if(err) return err; } - // Cast record elements - for(let i=0; i= 0; - const containsQuote = (quote !== '') && value.indexOf(quote) >= 0; - const containsEscape = value.indexOf(escape) >= 0 && (escape !== quote); - const containsRecordDelimiter = value.indexOf(record_delimiter) >= 0; - const quotedString = quoted_string && typeof field === 'string'; - let quotedMatch = quoted_match && quoted_match.filter(quoted_match => { - if(typeof quoted_match === 'string'){ - return value.indexOf(quoted_match) !== -1; - }else { - return quoted_match.test(value); + } + let csvrecord = ''; + for(let i=0; i 0; - const shouldQuote = containsQuote === true || containsdelimiter || containsRecordDelimiter || quoted || quotedString || quotedMatch; - if(shouldQuote === true && containsEscape === true){ - const regexp = escape === '\\' - ? new RegExp(escape + escape, 'g') - : new RegExp(escape, 'g'); - value = value.replace(regexp, escape + escape); + options = {...this.options, ...options}; + [err, options] = normalize_options(options); + if(err !== undefined){ + return [err]; + } + }else if(value === undefined || value === null){ + options = this.options; + }else { + return [Error(`Invalid Casting Value: returned value must return a string, an object, null or undefined, got ${JSON.stringify(value)}`)]; } - if(containsQuote === true){ - const regexp = new RegExp(quote,'g'); - value = value.replace(regexp, escape + quote); + const {delimiter, escape, quote, quoted, quoted_empty, quoted_string, quoted_match, record_delimiter} = options; + if(value){ + if(typeof value !== 'string'){ + return [Error(`Formatter must return a string, null or undefined, got ${JSON.stringify(value)}`)]; + } + const containsdelimiter = delimiter.length && value.indexOf(delimiter) >= 0; + const containsQuote = (quote !== '') && value.indexOf(quote) >= 0; + const containsEscape = value.indexOf(escape) >= 0 && (escape !== quote); + const containsRecordDelimiter = value.indexOf(record_delimiter) >= 0; + const quotedString = quoted_string && typeof field === 'string'; + let quotedMatch = quoted_match && quoted_match.filter(quoted_match => { + if(typeof quoted_match === 'string'){ + return value.indexOf(quoted_match) !== -1; + }else { + return quoted_match.test(value); + } + }); + quotedMatch = quotedMatch && quotedMatch.length > 0; + const shouldQuote = containsQuote === true || containsdelimiter || containsRecordDelimiter || quoted || quotedString || quotedMatch; + if(shouldQuote === true && containsEscape === true){ + const regexp = escape === '\\' + ? new RegExp(escape + escape, 'g') + : new RegExp(escape, 'g'); + value = value.replace(regexp, escape + escape); + } + if(containsQuote === true){ + const regexp = new RegExp(quote,'g'); + value = value.replace(regexp, escape + quote); + } + if(shouldQuote === true){ + value = quote + value + quote; + } + csvrecord += value; + }else if(quoted_empty === true || (field === '' && quoted_string === true && quoted_empty !== false)){ + csvrecord += quote + quote; } - if(shouldQuote === true){ - value = quote + value + quote; + if(i !== record.length - 1){ + csvrecord += delimiter; } - csvrecord += value; - }else if(quoted_empty === true || (field === '' && quoted_string === true && quoted_empty !== false)){ - csvrecord += quote + quote; } - if(i !== record.length - 1){ - csvrecord += delimiter; + return [undefined, csvrecord]; + }, + bom: function(push){ + if(this.options.bom !== true){ + return; } - } - return [undefined, csvrecord]; - } - bom(){ - if(this.options.bom !== true){ - return; - } - this.push(bom_utf8); - } - headers(){ - if(this.options.header === false){ - return; - } - if(this.options.columns === undefined){ - return; - } - let err; - let headers = this.options.columns.map(column => column.header); - if(this.options.eof){ - [err, headers] = this.stringify(headers, true); - headers += this.options.record_delimiter; - }else { - [err, headers] = this.stringify(headers); - } - if(err) return err; - this.push(headers); - } - __cast(value, context){ - const type = typeof value; - try{ - if(type === 'string'){ // Fine for 99% of the cases - return [undefined, this.options.cast.string(value, context)]; - }else if(type === 'bigint'){ - return [undefined, this.options.cast.bigint(value, context)]; - }else if(type === 'number'){ - return [undefined, this.options.cast.number(value, context)]; - }else if(type === 'boolean'){ - return [undefined, this.options.cast.boolean(value, context)]; - }else if(value instanceof Date){ - return [undefined, this.options.cast.date(value, context)]; - }else if(type === 'object' && value !== null){ - return [undefined, this.options.cast.object(value, context)]; - }else { - return [undefined, value, value]; + push(bom_utf8); + }, + headers: function(push){ + if(this.options.header === false){ + return; } - }catch(err){ - return [err]; - } - } - normalize_columns(columns){ - if(columns === undefined || columns === null){ - return []; - } - if(typeof columns !== 'object'){ - return [Error('Invalid option "columns": expect an array or an object')]; - } - if(!Array.isArray(columns)){ - const newcolumns = []; - for(const k in columns){ - newcolumns.push({ - key: k, - header: columns[k] - }); + if(this.options.columns === undefined){ + return; } - columns = newcolumns; - }else { - const newcolumns = []; - for(const column of columns){ - if(typeof column === 'string'){ - newcolumns.push({ - key: column, - header: column - }); - }else if(typeof column === 'object' && column !== undefined && !Array.isArray(column)){ - if(!column.key){ - return [Error('Invalid column definition: property "key" is required')]; - } - if(column.header === undefined){ - column.header = column.key; - } - newcolumns.push(column); + let err; + let headers = this.options.columns.map(column => column.header); + if(this.options.eof){ + [err, headers] = this.stringify(headers, true); + headers += this.options.record_delimiter; + }else { + [err, headers] = this.stringify(headers); + } + if(err) return err; + push(headers); + }, + __cast: function(value, context){ + const type = typeof value; + try{ + if(type === 'string'){ // Fine for 99% of the cases + return [undefined, this.options.cast.string(value, context)]; + }else if(type === 'bigint'){ + return [undefined, this.options.cast.bigint(value, context)]; + }else if(type === 'number'){ + return [undefined, this.options.cast.number(value, context)]; + }else if(type === 'boolean'){ + return [undefined, this.options.cast.boolean(value, context)]; + }else if(value instanceof Date){ + return [undefined, this.options.cast.date(value, context)]; + }else if(type === 'object' && value !== null){ + return [undefined, this.options.cast.object(value, context)]; }else { - return [Error('Invalid column definition: expect a string or an object')]; + return [undefined, value, value]; } + }catch(err){ + return [err]; } - columns = newcolumns; } - return [undefined, columns]; - } - } + }; + }; - const stringify = function(records, options={}){ + const stringify = function(records, opts={}){ const data = []; - const stringifier = new Stringifier(options); - stringifier.push = function(record){ - if(record === null){ - return; - } - data.push(record.toString()); + const [err, options] = normalize_options(opts); + if(err !== undefined) throw err; + const state = { + stop: false + }; + // Information + const info = { + records: 0 }; + const api = stringifier(options, state, info); + // stringifier.push = function(record){ + // if(record === null){ + // return; + // } + // data.push(record.toString()); + // }; for(const record of records){ - const err = stringifier.__transform(record, null); + const err = api.__transform(record, function(record){ + data.push(record); + }); if(err !== undefined) throw err; } return data.join(''); diff --git a/packages/csv/dist/umd/index.js b/packages/csv/dist/umd/index.js index 792fcb39c..b72ab5fd3 100644 --- a/packages/csv/dist/umd/index.js +++ b/packages/csv/dist/umd/index.js @@ -118,7 +118,7 @@ return parts.join('') } - function read (buffer, offset, isLE, mLen, nBytes) { + function read$1 (buffer, offset, isLE, mLen, nBytes) { var e, m; var eLen = nBytes * 8 - mLen - 1; var eMax = (1 << eLen) - 1; @@ -1399,22 +1399,22 @@ Buffer.prototype.readFloatLE = function readFloatLE (offset, noAssert) { if (!noAssert) checkOffset(offset, 4, this.length); - return read(this, offset, true, 23, 4) + return read$1(this, offset, true, 23, 4) }; Buffer.prototype.readFloatBE = function readFloatBE (offset, noAssert) { if (!noAssert) checkOffset(offset, 4, this.length); - return read(this, offset, false, 23, 4) + return read$1(this, offset, false, 23, 4) }; Buffer.prototype.readDoubleLE = function readDoubleLE (offset, noAssert) { if (!noAssert) checkOffset(offset, 8, this.length); - return read(this, offset, true, 52, 8) + return read$1(this, offset, true, 52, 8) }; Buffer.prototype.readDoubleBE = function readDoubleBE (offset, noAssert) { if (!noAssert) checkOffset(offset, 8, this.length); - return read(this, offset, false, 52, 8) + return read$1(this, offset, false, 52, 8) }; function checkInt (buf, value, offset, ext, max, min) { @@ -2633,7 +2633,7 @@ } }); for (var x = args[i]; i < len; x = args[++i]) { - if (isNull(x) || !isObject$2(x)) { + if (isNull(x) || !isObject(x)) { str += ' ' + x; } else { str += ' ' + inspect(x); @@ -3049,19 +3049,19 @@ } function isRegExp(re) { - return isObject$2(re) && objectToString(re) === '[object RegExp]'; + return isObject(re) && objectToString(re) === '[object RegExp]'; } - function isObject$2(arg) { + function isObject(arg) { return typeof arg === 'object' && arg !== null; } function isDate(d) { - return isObject$2(d) && objectToString(d) === '[object Date]'; + return isObject(d) && objectToString(d) === '[object Date]'; } function isError(e) { - return isObject$2(e) && + return isObject(e) && (objectToString(e) === '[object Error]' || e instanceof Error); } @@ -3112,7 +3112,7 @@ function _extend(origin, add) { // Don't do anything if add isn't an object - if (!add || !isObject$2(add)) return origin; + if (!add || !isObject(add)) return origin; var keys = Object.keys(add); var i = keys.length; @@ -3134,7 +3134,7 @@ isFunction: isFunction, isError: isError, isDate: isDate, - isObject: isObject$2, + isObject: isObject, isRegExp: isRegExp, isUndefined: isUndefined, isSymbol: isSymbol$1, @@ -5042,20 +5042,64 @@ return dest; }; - const Generator = function(options = {}){ + const init_state$1 = (options) => { + // State + return { + start_time: options.duration ? Date.now() : null, + fixed_size_buffer: '', + count_written: 0, + count_created: 0, + }; + }; + + // Generate a random number between 0 and 1 with 2 decimals. The function is idempotent if it detect the "seed" option. + const random = function(options={}){ + if(options.seed){ + return options.seed = options.seed * Math.PI * 100 % 100 / 100; + }else { + return Math.random(); + } + }; + + const types = { + // Generate an ASCII value. + ascii: function({options}){ + const column = []; + const nb_chars = Math.ceil(random(options) * options.maxWordLength); + for(let i=0; i { // Convert Stream Readable options if underscored - if(options.high_water_mark){ - options.highWaterMark = options.high_water_mark; + if(opts.high_water_mark){ + opts.highWaterMark = opts.high_water_mark; } - if(options.object_mode){ - options.objectMode = options.object_mode; + if(opts.object_mode){ + opts.objectMode = opts.object_mode; } - // Call parent constructor - Stream.Readable.call(this, options); // Clone and camelize options - this.options = {}; - for(const k in options){ - this.options[Generator.camelize(k)] = options[k]; + const options = {}; + for(const k in opts){ + options[camelize(k)] = opts[k]; } // Normalize options const dft = { @@ -5073,110 +5117,100 @@ sleep: 0, }; for(const k in dft){ - if(this.options[k] === undefined){ - this.options[k] = dft[k]; + if(options[k] === undefined){ + options[k] = dft[k]; } } // Default values - if(this.options.eof === true){ - this.options.eof = this.options.rowDelimiter; + if(options.eof === true){ + options.eof = options.rowDelimiter; } - // State - this._ = { - start_time: this.options.duration ? Date.now() : null, - fixed_size_buffer: '', - count_written: 0, - count_created: 0, - }; - if(typeof this.options.columns === 'number'){ - this.options.columns = new Array(this.options.columns); + if(typeof options.columns === 'number'){ + options.columns = new Array(options.columns); } - const accepted_header_types = Object.keys(Generator).filter((t) => (!['super_', 'camelize'].includes(t))); - for(let i = 0; i < this.options.columns.length; i++){ - const v = this.options.columns[i] || 'ascii'; + const accepted_header_types = Object.keys(types).filter((t) => (!['super_', 'camelize'].includes(t))); + for(let i = 0; i < options.columns.length; i++){ + const v = options.columns[i] || 'ascii'; if(typeof v === 'string'){ if(!accepted_header_types.includes(v)){ throw Error(`Invalid column type: got "${v}", default values are ${JSON.stringify(accepted_header_types)}`); } - this.options.columns[i] = Generator[v]; + options.columns[i] = types[v]; } } - return this; + return options; }; - util.inherits(Generator, Stream.Readable); - // Generate a random number between 0 and 1 with 2 decimals. The function is idempotent if it detect the "seed" option. - Generator.prototype.random = function(){ - if(this.options.seed){ - return this.options.seed = this.options.seed * Math.PI * 100 % 100 / 100; - }else { - return Math.random(); - } - }; - // Stop the generation. - Generator.prototype.end = function(){ - this.push(null); - }; - // Put new data into the read queue. - Generator.prototype._read = function(size){ + const read = (options, state, size, push, close) => { // Already started const data = []; - let length = this._.fixed_size_buffer.length; + let length = state.fixed_size_buffer.length; if(length !== 0){ - data.push(this._.fixed_size_buffer); + data.push(state.fixed_size_buffer); } // eslint-disable-next-line while(true){ // Time for some rest: flush first and stop later - if((this._.count_created === this.options.length) || (this.options.end && Date.now() > this.options.end) || (this.options.duration && Date.now() > this._.start_time + this.options.duration)){ + if((state.count_created === options.length) || (options.end && Date.now() > options.end) || (options.duration && Date.now() > state.start_time + options.duration)){ // Flush if(data.length){ - if(this.options.objectMode){ + if(options.objectMode){ for(const record of data){ - this.__push(record); + push(record); } }else { - this.__push(data.join('') + (this.options.eof ? this.options.eof : '')); + push(data.join('') + (options.eof ? options.eof : '')); } - this._.end = true; + state.end = true; }else { - this.push(null); + close(); } return; } // Create the record let record = []; let recordLength; - this.options.columns.forEach((fn) => { - record.push(fn(this)); + options.columns.forEach((fn) => { + const result = fn({options: options, state: state}); + const type = typeof result; + if(result !== null && type !== 'string' && type !== 'number'){ + throw Error([ + 'INVALID_VALUE:', + 'values returned by column function must be', + 'a string, a number or null,', + `got ${JSON.stringify(result)}` + ].join(' ')); + } + record.push(result); }); // Obtain record length - if(this.options.objectMode){ + if(options.objectMode){ recordLength = 0; // recordLength is currently equal to the number of columns // This is wrong and shall equal to 1 record only - for(const column of record) + for(const column of record){ recordLength += column.length; + } }else { // Stringify the record - record = (this._.count_created === 0 ? '' : this.options.rowDelimiter)+record.join(this.options.delimiter); + record = (state.count_created === 0 ? '' : options.rowDelimiter)+record.join(options.delimiter); recordLength = record.length; } - this._.count_created++; + state.count_created++; if(length + recordLength > size){ - if(this.options.objectMode){ + if(options.objectMode){ data.push(record); for(const record of data){ - this.__push(record); + push(record); } }else { - if(this.options.fixedSize){ - this._.fixed_size_buffer = record.substr(size - length); + if(options.fixedSize){ + state.fixed_size_buffer = record.substr(size - length); data.push(record.substr(0, size - length)); }else { data.push(record); } - this.__push(data.join('')); + push(data.join('')); } return; } @@ -5184,42 +5218,41 @@ data.push(record); } }; + + const Generator = function(options = {}){ + this.options = normalize_options$2(options); + // Call parent constructor + Stream.Readable.call(this, this.options); + this.state = init_state$1(this.options); + return this; + }; + util.inherits(Generator, Stream.Readable); + + // Stop the generation. + Generator.prototype.end = function(){ + this.push(null); + }; + // Put new data into the read queue. + Generator.prototype._read = function(size){ + const self = this; + read(this.options, this.state, size, function(chunk) { + self.__push(chunk); + }, function(){ + self.push(null); + }); + }; // Put new data into the read queue. Generator.prototype.__push = function(record){ + // console.log('push', record) const push = () => { - this._.count_written++; + this.state.count_written++; this.push(record); - if(this._.end === true){ + if(this.state.end === true){ return this.push(null); } }; this.options.sleep > 0 ? setTimeout(push, this.options.sleep) : push(); }; - // Generate an ASCII value. - Generator.ascii = function(gen){ - // Column - const column = []; - const nb_chars = Math.ceil(gen.random() * gen.options.maxWordLength); - for(let i=0; i delimiter.length), + // Skip if the remaining buffer can be escape sequence + options.quote !== null ? options.quote.length : 0, + ), + previousBuf: undefined, + quoting: false, + stop: false, + rawBuffer: new ResizeableBuffer(100), + record: [], + recordHasError: false, + record_length: 0, + recordDelimiterMaxLength: options.record_delimiter.length === 0 ? 2 : Math.max(...options.record_delimiter.map((v) => v.length)), + trimChars: [Buffer.from(' ', options.encoding)[0], Buffer.from('\t', options.encoding)[0]], + wasQuoting: false, + wasRowDelimiter: false + }; }; - class CsvError$1 extends Error { - constructor(code, message, options, ...contexts) { - if(Array.isArray(message)) message = message.join(' '); - super(message); - if(Error.captureStackTrace !== undefined){ - Error.captureStackTrace(this, CsvError$1); - } - this.code = code; - for(const context of contexts){ - for(const key in context){ - const value = context[key]; - this[key] = isBuffer$1(value) ? value.toString(options.encoding) : value == null ? value : JSON.parse(JSON.stringify(value)); - } - } - } - } - const underscore$1 = function(str){ return str.replace(/([A-Z])/g, function(_, match){ return '_' + match.toLowerCase(); }); }; - const isObject$1 = function(obj){ - return (typeof obj === 'object' && obj !== null && !Array.isArray(obj)); - }; - - const isRecordEmpty = function(record){ - return record.every((field) => field == null || field.toString && field.toString().trim() === ''); - }; - - const normalizeColumnsArray = function(columns){ - const normalizedColumns = []; - for(let i = 0, l = columns.length; i < l; i++){ - const column = columns[i]; - if(column === undefined || column === null || column === false){ - normalizedColumns[i] = { disabled: true }; - }else if(typeof column === 'string'){ - normalizedColumns[i] = { name: column }; - }else if(isObject$1(column)){ - if(typeof column.name !== 'string'){ - throw new CsvError$1('CSV_OPTION_COLUMNS_MISSING_NAME', [ - 'Option columns missing name:', - `property "name" is required at position ${i}`, - 'when column is an object literal' - ]); - } - normalizedColumns[i] = column; - }else { - throw new CsvError$1('CSV_INVALID_COLUMN_DEFINITION', [ - 'Invalid column definition:', - 'expect a string or a literal object,', - `got ${JSON.stringify(column)} at position ${i}` - ]); - } - } - return normalizedColumns; - }; - - class Parser extends Transform { - constructor(opts = {}){ - super({...{readableObjectMode: true}, ...opts, encoding: null}); - this.__originalOptions = opts; - this.__normalizeOptions(opts); - } - __normalizeOptions(opts){ - const options = {}; - // Merge with user options - for(const opt in opts){ - options[underscore$1(opt)] = opts[opt]; - } - // Normalize option `encoding` - // Note: defined first because other options depends on it - // to convert chars/strings into buffers. - if(options.encoding === undefined || options.encoding === true){ - options.encoding = 'utf8'; - }else if(options.encoding === null || options.encoding === false){ - options.encoding = null; - }else if(typeof options.encoding !== 'string' && options.encoding !== null){ - throw new CsvError$1('CSV_INVALID_OPTION_ENCODING', [ - 'Invalid option encoding:', - 'encoding must be a string or null to return a buffer,', - `got ${JSON.stringify(options.encoding)}` - ], options); - } - // Normalize option `bom` - if(options.bom === undefined || options.bom === null || options.bom === false){ - options.bom = false; - }else if(options.bom !== true){ - throw new CsvError$1('CSV_INVALID_OPTION_BOM', [ - 'Invalid option bom:', 'bom must be true,', - `got ${JSON.stringify(options.bom)}` - ], options); - } - // Normalize option `cast` - let fnCastField = null; - if(options.cast === undefined || options.cast === null || options.cast === false || options.cast === ''){ - options.cast = undefined; - }else if(typeof options.cast === 'function'){ - fnCastField = options.cast; - options.cast = true; - }else if(options.cast !== true){ - throw new CsvError$1('CSV_INVALID_OPTION_CAST', [ - 'Invalid option cast:', 'cast must be true or a function,', - `got ${JSON.stringify(options.cast)}` - ], options); - } - // Normalize option `cast_date` - if(options.cast_date === undefined || options.cast_date === null || options.cast_date === false || options.cast_date === ''){ - options.cast_date = false; - }else if(options.cast_date === true){ - options.cast_date = function(value){ - const date = Date.parse(value); - return !isNaN(date) ? new Date(date) : value; - }; - }else { - throw new CsvError$1('CSV_INVALID_OPTION_CAST_DATE', [ - 'Invalid option cast_date:', 'cast_date must be true or a function,', - `got ${JSON.stringify(options.cast_date)}` - ], options); - } - // Normalize option `columns` - let fnFirstLineToHeaders = null; - if(options.columns === true){ - // Fields in the first line are converted as-is to columns - fnFirstLineToHeaders = undefined; - }else if(typeof options.columns === 'function'){ - fnFirstLineToHeaders = options.columns; - options.columns = true; - }else if(Array.isArray(options.columns)){ - options.columns = normalizeColumnsArray(options.columns); - }else if(options.columns === undefined || options.columns === null || options.columns === false){ - options.columns = false; - }else { - throw new CsvError$1('CSV_INVALID_OPTION_COLUMNS', [ - 'Invalid option columns:', - 'expect an array, a function or true,', - `got ${JSON.stringify(options.columns)}` + const normalize_options$1 = function(opts){ + const options = {}; + // Merge with user options + for(const opt in opts){ + options[underscore$1(opt)] = opts[opt]; + } + // Normalize option `encoding` + // Note: defined first because other options depends on it + // to convert chars/strings into buffers. + if(options.encoding === undefined || options.encoding === true){ + options.encoding = 'utf8'; + }else if(options.encoding === null || options.encoding === false){ + options.encoding = null; + }else if(typeof options.encoding !== 'string' && options.encoding !== null){ + throw new CsvError$1('CSV_INVALID_OPTION_ENCODING', [ + 'Invalid option encoding:', + 'encoding must be a string or null to return a buffer,', + `got ${JSON.stringify(options.encoding)}` + ], options); + } + // Normalize option `bom` + if(options.bom === undefined || options.bom === null || options.bom === false){ + options.bom = false; + }else if(options.bom !== true){ + throw new CsvError$1('CSV_INVALID_OPTION_BOM', [ + 'Invalid option bom:', 'bom must be true,', + `got ${JSON.stringify(options.bom)}` + ], options); + } + // Normalize option `cast` + options.cast_function = null; + if(options.cast === undefined || options.cast === null || options.cast === false || options.cast === ''){ + options.cast = undefined; + }else if(typeof options.cast === 'function'){ + options.cast_function = options.cast; + options.cast = true; + }else if(options.cast !== true){ + throw new CsvError$1('CSV_INVALID_OPTION_CAST', [ + 'Invalid option cast:', 'cast must be true or a function,', + `got ${JSON.stringify(options.cast)}` + ], options); + } + // Normalize option `cast_date` + if(options.cast_date === undefined || options.cast_date === null || options.cast_date === false || options.cast_date === ''){ + options.cast_date = false; + }else if(options.cast_date === true){ + options.cast_date = function(value){ + const date = Date.parse(value); + return !isNaN(date) ? new Date(date) : value; + }; + }else { + throw new CsvError$1('CSV_INVALID_OPTION_CAST_DATE', [ + 'Invalid option cast_date:', 'cast_date must be true or a function,', + `got ${JSON.stringify(options.cast_date)}` + ], options); + } + // Normalize option `columns` + options.cast_first_line_to_header = null; + if(options.columns === true){ + // Fields in the first line are converted as-is to columns + options.cast_first_line_to_header = undefined; + }else if(typeof options.columns === 'function'){ + options.cast_first_line_to_header = options.columns; + options.columns = true; + }else if(Array.isArray(options.columns)){ + options.columns = normalize_columns_array(options.columns); + }else if(options.columns === undefined || options.columns === null || options.columns === false){ + options.columns = false; + }else { + throw new CsvError$1('CSV_INVALID_OPTION_COLUMNS', [ + 'Invalid option columns:', + 'expect an array, a function or true,', + `got ${JSON.stringify(options.columns)}` + ], options); + } + // Normalize option `group_columns_by_name` + if(options.group_columns_by_name === undefined || options.group_columns_by_name === null || options.group_columns_by_name === false){ + options.group_columns_by_name = false; + }else if(options.group_columns_by_name !== true){ + throw new CsvError$1('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ + 'Invalid option group_columns_by_name:', + 'expect an boolean,', + `got ${JSON.stringify(options.group_columns_by_name)}` + ], options); + }else if(options.columns === false){ + throw new CsvError$1('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ + 'Invalid option group_columns_by_name:', + 'the `columns` mode must be activated.' + ], options); + } + // Normalize option `comment` + if(options.comment === undefined || options.comment === null || options.comment === false || options.comment === ''){ + options.comment = null; + }else { + if(typeof options.comment === 'string'){ + options.comment = Buffer.from(options.comment, options.encoding); + } + if(!isBuffer$1(options.comment)){ + throw new CsvError$1('CSV_INVALID_OPTION_COMMENT', [ + 'Invalid option comment:', + 'comment must be a buffer or a string,', + `got ${JSON.stringify(options.comment)}` ], options); } - // Normalize option `group_columns_by_name` - if(options.group_columns_by_name === undefined || options.group_columns_by_name === null || options.group_columns_by_name === false){ - options.group_columns_by_name = false; - }else if(options.group_columns_by_name !== true){ - throw new CsvError$1('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ - 'Invalid option group_columns_by_name:', - 'expect an boolean,', - `got ${JSON.stringify(options.group_columns_by_name)}` - ], options); - }else if(options.columns === false){ - throw new CsvError$1('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ - 'Invalid option group_columns_by_name:', - 'the `columns` mode must be activated.' - ], options); + } + // Normalize option `delimiter` + const delimiter_json = JSON.stringify(options.delimiter); + if(!Array.isArray(options.delimiter)) options.delimiter = [options.delimiter]; + if(options.delimiter.length === 0){ + throw new CsvError$1('CSV_INVALID_OPTION_DELIMITER', [ + 'Invalid option delimiter:', + 'delimiter must be a non empty string or buffer or array of string|buffer,', + `got ${delimiter_json}` + ], options); + } + options.delimiter = options.delimiter.map(function(delimiter){ + if(delimiter === undefined || delimiter === null || delimiter === false){ + return Buffer.from(',', options.encoding); } - // Normalize option `comment` - if(options.comment === undefined || options.comment === null || options.comment === false || options.comment === ''){ - options.comment = null; - }else { - if(typeof options.comment === 'string'){ - options.comment = Buffer.from(options.comment, options.encoding); - } - if(!isBuffer$1(options.comment)){ - throw new CsvError$1('CSV_INVALID_OPTION_COMMENT', [ - 'Invalid option comment:', - 'comment must be a buffer or a string,', - `got ${JSON.stringify(options.comment)}` - ], options); - } + if(typeof delimiter === 'string'){ + delimiter = Buffer.from(delimiter, options.encoding); } - // Normalize option `delimiter` - const delimiter_json = JSON.stringify(options.delimiter); - if(!Array.isArray(options.delimiter)) options.delimiter = [options.delimiter]; - if(options.delimiter.length === 0){ + if(!isBuffer$1(delimiter) || delimiter.length === 0){ throw new CsvError$1('CSV_INVALID_OPTION_DELIMITER', [ 'Invalid option delimiter:', 'delimiter must be a non empty string or buffer or array of string|buffer,', `got ${delimiter_json}` ], options); } - options.delimiter = options.delimiter.map(function(delimiter){ - if(delimiter === undefined || delimiter === null || delimiter === false){ - return Buffer.from(',', options.encoding); - } - if(typeof delimiter === 'string'){ - delimiter = Buffer.from(delimiter, options.encoding); - } - if(!isBuffer$1(delimiter) || delimiter.length === 0){ - throw new CsvError$1('CSV_INVALID_OPTION_DELIMITER', [ - 'Invalid option delimiter:', - 'delimiter must be a non empty string or buffer or array of string|buffer,', - `got ${delimiter_json}` - ], options); - } - return delimiter; - }); - // Normalize option `escape` - if(options.escape === undefined || options.escape === true){ - options.escape = Buffer.from('"', options.encoding); - }else if(typeof options.escape === 'string'){ - options.escape = Buffer.from(options.escape, options.encoding); - }else if (options.escape === null || options.escape === false){ - options.escape = null; - } - if(options.escape !== null){ - if(!isBuffer$1(options.escape)){ - throw new Error(`Invalid Option: escape must be a buffer, a string or a boolean, got ${JSON.stringify(options.escape)}`); - } + return delimiter; + }); + // Normalize option `escape` + if(options.escape === undefined || options.escape === true){ + options.escape = Buffer.from('"', options.encoding); + }else if(typeof options.escape === 'string'){ + options.escape = Buffer.from(options.escape, options.encoding); + }else if (options.escape === null || options.escape === false){ + options.escape = null; + } + if(options.escape !== null){ + if(!isBuffer$1(options.escape)){ + throw new Error(`Invalid Option: escape must be a buffer, a string or a boolean, got ${JSON.stringify(options.escape)}`); + } + } + // Normalize option `from` + if(options.from === undefined || options.from === null){ + options.from = 1; + }else { + if(typeof options.from === 'string' && /\d+/.test(options.from)){ + options.from = parseInt(options.from); } - // Normalize option `from` - if(options.from === undefined || options.from === null){ - options.from = 1; - }else { - if(typeof options.from === 'string' && /\d+/.test(options.from)){ - options.from = parseInt(options.from); - } - if(Number.isInteger(options.from)){ - if(options.from < 0){ - throw new Error(`Invalid Option: from must be a positive integer, got ${JSON.stringify(opts.from)}`); - } - }else { - throw new Error(`Invalid Option: from must be an integer, got ${JSON.stringify(options.from)}`); + if(Number.isInteger(options.from)){ + if(options.from < 0){ + throw new Error(`Invalid Option: from must be a positive integer, got ${JSON.stringify(opts.from)}`); } - } - // Normalize option `from_line` - if(options.from_line === undefined || options.from_line === null){ - options.from_line = 1; }else { - if(typeof options.from_line === 'string' && /\d+/.test(options.from_line)){ - options.from_line = parseInt(options.from_line); - } - if(Number.isInteger(options.from_line)){ - if(options.from_line <= 0){ - throw new Error(`Invalid Option: from_line must be a positive integer greater than 0, got ${JSON.stringify(opts.from_line)}`); - } - }else { - throw new Error(`Invalid Option: from_line must be an integer, got ${JSON.stringify(opts.from_line)}`); - } - } - // Normalize options `ignore_last_delimiters` - if(options.ignore_last_delimiters === undefined || options.ignore_last_delimiters === null){ - options.ignore_last_delimiters = false; - }else if(typeof options.ignore_last_delimiters === 'number'){ - options.ignore_last_delimiters = Math.floor(options.ignore_last_delimiters); - if(options.ignore_last_delimiters === 0){ - options.ignore_last_delimiters = false; - } - }else if(typeof options.ignore_last_delimiters !== 'boolean'){ - throw new CsvError$1('CSV_INVALID_OPTION_IGNORE_LAST_DELIMITERS', [ - 'Invalid option `ignore_last_delimiters`:', - 'the value must be a boolean value or an integer,', - `got ${JSON.stringify(options.ignore_last_delimiters)}` - ], options); + throw new Error(`Invalid Option: from must be an integer, got ${JSON.stringify(options.from)}`); } - if(options.ignore_last_delimiters === true && options.columns === false){ - throw new CsvError$1('CSV_IGNORE_LAST_DELIMITERS_REQUIRES_COLUMNS', [ - 'The option `ignore_last_delimiters`', - 'requires the activation of the `columns` option' - ], options); + } + // Normalize option `from_line` + if(options.from_line === undefined || options.from_line === null){ + options.from_line = 1; + }else { + if(typeof options.from_line === 'string' && /\d+/.test(options.from_line)){ + options.from_line = parseInt(options.from_line); } - // Normalize option `info` - if(options.info === undefined || options.info === null || options.info === false){ - options.info = false; - }else if(options.info !== true){ - throw new Error(`Invalid Option: info must be true, got ${JSON.stringify(options.info)}`); - } - // Normalize option `max_record_size` - if(options.max_record_size === undefined || options.max_record_size === null || options.max_record_size === false){ - options.max_record_size = 0; - }else if(Number.isInteger(options.max_record_size) && options.max_record_size >= 0);else if(typeof options.max_record_size === 'string' && /\d+/.test(options.max_record_size)){ - options.max_record_size = parseInt(options.max_record_size); - }else { - throw new Error(`Invalid Option: max_record_size must be a positive integer, got ${JSON.stringify(options.max_record_size)}`); - } - // Normalize option `objname` - if(options.objname === undefined || options.objname === null || options.objname === false){ - options.objname = undefined; - }else if(isBuffer$1(options.objname)){ - if(options.objname.length === 0){ - throw new Error(`Invalid Option: objname must be a non empty buffer`); - } - if(options.encoding === null);else { - options.objname = options.objname.toString(options.encoding); - } - }else if(typeof options.objname === 'string'){ - if(options.objname.length === 0){ - throw new Error(`Invalid Option: objname must be a non empty string`); - } - // Great, nothing to do - }else if(typeof options.objname === 'number');else { - throw new Error(`Invalid Option: objname must be a string or a buffer, got ${options.objname}`); - } - if(options.objname !== undefined){ - if(typeof options.objname === 'number'){ - if(options.columns !== false){ - throw Error('Invalid Option: objname index cannot be combined with columns or be defined as a field'); - } - }else { // A string or a buffer - if(options.columns === false){ - throw Error('Invalid Option: objname field must be combined with columns or be defined as an index'); - } + if(Number.isInteger(options.from_line)){ + if(options.from_line <= 0){ + throw new Error(`Invalid Option: from_line must be a positive integer greater than 0, got ${JSON.stringify(opts.from_line)}`); } + }else { + throw new Error(`Invalid Option: from_line must be an integer, got ${JSON.stringify(opts.from_line)}`); } - // Normalize option `on_record` - if(options.on_record === undefined || options.on_record === null){ - options.on_record = undefined; - }else if(typeof options.on_record !== 'function'){ - throw new CsvError$1('CSV_INVALID_OPTION_ON_RECORD', [ - 'Invalid option `on_record`:', - 'expect a function,', - `got ${JSON.stringify(options.on_record)}` - ], options); + } + // Normalize options `ignore_last_delimiters` + if(options.ignore_last_delimiters === undefined || options.ignore_last_delimiters === null){ + options.ignore_last_delimiters = false; + }else if(typeof options.ignore_last_delimiters === 'number'){ + options.ignore_last_delimiters = Math.floor(options.ignore_last_delimiters); + if(options.ignore_last_delimiters === 0){ + options.ignore_last_delimiters = false; } - // Normalize option `quote` - if(options.quote === null || options.quote === false || options.quote === ''){ - options.quote = null; - }else { - if(options.quote === undefined || options.quote === true){ - options.quote = Buffer.from('"', options.encoding); - }else if(typeof options.quote === 'string'){ - options.quote = Buffer.from(options.quote, options.encoding); + }else if(typeof options.ignore_last_delimiters !== 'boolean'){ + throw new CsvError$1('CSV_INVALID_OPTION_IGNORE_LAST_DELIMITERS', [ + 'Invalid option `ignore_last_delimiters`:', + 'the value must be a boolean value or an integer,', + `got ${JSON.stringify(options.ignore_last_delimiters)}` + ], options); + } + if(options.ignore_last_delimiters === true && options.columns === false){ + throw new CsvError$1('CSV_IGNORE_LAST_DELIMITERS_REQUIRES_COLUMNS', [ + 'The option `ignore_last_delimiters`', + 'requires the activation of the `columns` option' + ], options); + } + // Normalize option `info` + if(options.info === undefined || options.info === null || options.info === false){ + options.info = false; + }else if(options.info !== true){ + throw new Error(`Invalid Option: info must be true, got ${JSON.stringify(options.info)}`); + } + // Normalize option `max_record_size` + if(options.max_record_size === undefined || options.max_record_size === null || options.max_record_size === false){ + options.max_record_size = 0; + }else if(Number.isInteger(options.max_record_size) && options.max_record_size >= 0);else if(typeof options.max_record_size === 'string' && /\d+/.test(options.max_record_size)){ + options.max_record_size = parseInt(options.max_record_size); + }else { + throw new Error(`Invalid Option: max_record_size must be a positive integer, got ${JSON.stringify(options.max_record_size)}`); + } + // Normalize option `objname` + if(options.objname === undefined || options.objname === null || options.objname === false){ + options.objname = undefined; + }else if(isBuffer$1(options.objname)){ + if(options.objname.length === 0){ + throw new Error(`Invalid Option: objname must be a non empty buffer`); + } + if(options.encoding === null);else { + options.objname = options.objname.toString(options.encoding); + } + }else if(typeof options.objname === 'string'){ + if(options.objname.length === 0){ + throw new Error(`Invalid Option: objname must be a non empty string`); + } + // Great, nothing to do + }else if(typeof options.objname === 'number');else { + throw new Error(`Invalid Option: objname must be a string or a buffer, got ${options.objname}`); + } + if(options.objname !== undefined){ + if(typeof options.objname === 'number'){ + if(options.columns !== false){ + throw Error('Invalid Option: objname index cannot be combined with columns or be defined as a field'); } - if(!isBuffer$1(options.quote)){ - throw new Error(`Invalid Option: quote must be a buffer or a string, got ${JSON.stringify(options.quote)}`); + }else { // A string or a buffer + if(options.columns === false){ + throw Error('Invalid Option: objname field must be combined with columns or be defined as an index'); } } - // Normalize option `raw` - if(options.raw === undefined || options.raw === null || options.raw === false){ - options.raw = false; - }else if(options.raw !== true){ - throw new Error(`Invalid Option: raw must be true, got ${JSON.stringify(options.raw)}`); - } - // Normalize option `record_delimiter` - if(options.record_delimiter === undefined){ - options.record_delimiter = []; - }else if(typeof options.record_delimiter === 'string' || isBuffer$1(options.record_delimiter)){ - if(options.record_delimiter.length === 0){ - throw new CsvError$1('CSV_INVALID_OPTION_RECORD_DELIMITER', [ - 'Invalid option `record_delimiter`:', - 'value must be a non empty string or buffer,', - `got ${JSON.stringify(options.record_delimiter)}` - ], options); - } - options.record_delimiter = [options.record_delimiter]; - }else if(!Array.isArray(options.record_delimiter)){ + } + // Normalize option `on_record` + if(options.on_record === undefined || options.on_record === null){ + options.on_record = undefined; + }else if(typeof options.on_record !== 'function'){ + throw new CsvError$1('CSV_INVALID_OPTION_ON_RECORD', [ + 'Invalid option `on_record`:', + 'expect a function,', + `got ${JSON.stringify(options.on_record)}` + ], options); + } + // Normalize option `quote` + if(options.quote === null || options.quote === false || options.quote === ''){ + options.quote = null; + }else { + if(options.quote === undefined || options.quote === true){ + options.quote = Buffer.from('"', options.encoding); + }else if(typeof options.quote === 'string'){ + options.quote = Buffer.from(options.quote, options.encoding); + } + if(!isBuffer$1(options.quote)){ + throw new Error(`Invalid Option: quote must be a buffer or a string, got ${JSON.stringify(options.quote)}`); + } + } + // Normalize option `raw` + if(options.raw === undefined || options.raw === null || options.raw === false){ + options.raw = false; + }else if(options.raw !== true){ + throw new Error(`Invalid Option: raw must be true, got ${JSON.stringify(options.raw)}`); + } + // Normalize option `record_delimiter` + if(options.record_delimiter === undefined){ + options.record_delimiter = []; + }else if(typeof options.record_delimiter === 'string' || isBuffer$1(options.record_delimiter)){ + if(options.record_delimiter.length === 0){ throw new CsvError$1('CSV_INVALID_OPTION_RECORD_DELIMITER', [ 'Invalid option `record_delimiter`:', - 'value must be a string, a buffer or array of string|buffer,', + 'value must be a non empty string or buffer,', `got ${JSON.stringify(options.record_delimiter)}` ], options); } - options.record_delimiter = options.record_delimiter.map(function(rd, i){ - if(typeof rd !== 'string' && ! isBuffer$1(rd)){ - throw new CsvError$1('CSV_INVALID_OPTION_RECORD_DELIMITER', [ - 'Invalid option `record_delimiter`:', - 'value must be a string, a buffer or array of string|buffer', - `at index ${i},`, - `got ${JSON.stringify(rd)}` - ], options); - }else if(rd.length === 0){ - throw new CsvError$1('CSV_INVALID_OPTION_RECORD_DELIMITER', [ - 'Invalid option `record_delimiter`:', - 'value must be a non empty string or buffer', - `at index ${i},`, - `got ${JSON.stringify(rd)}` - ], options); - } - if(typeof rd === 'string'){ - rd = Buffer.from(rd, options.encoding); - } - return rd; - }); - // Normalize option `relax_column_count` - if(typeof options.relax_column_count === 'boolean');else if(options.relax_column_count === undefined || options.relax_column_count === null){ - options.relax_column_count = false; - }else { - throw new Error(`Invalid Option: relax_column_count must be a boolean, got ${JSON.stringify(options.relax_column_count)}`); - } - if(typeof options.relax_column_count_less === 'boolean');else if(options.relax_column_count_less === undefined || options.relax_column_count_less === null){ - options.relax_column_count_less = false; - }else { - throw new Error(`Invalid Option: relax_column_count_less must be a boolean, got ${JSON.stringify(options.relax_column_count_less)}`); - } - if(typeof options.relax_column_count_more === 'boolean');else if(options.relax_column_count_more === undefined || options.relax_column_count_more === null){ - options.relax_column_count_more = false; - }else { - throw new Error(`Invalid Option: relax_column_count_more must be a boolean, got ${JSON.stringify(options.relax_column_count_more)}`); - } - // Normalize option `relax_quotes` - if(typeof options.relax_quotes === 'boolean');else if(options.relax_quotes === undefined || options.relax_quotes === null){ - options.relax_quotes = false; - }else { - throw new Error(`Invalid Option: relax_quotes must be a boolean, got ${JSON.stringify(options.relax_quotes)}`); + options.record_delimiter = [options.record_delimiter]; + }else if(!Array.isArray(options.record_delimiter)){ + throw new CsvError$1('CSV_INVALID_OPTION_RECORD_DELIMITER', [ + 'Invalid option `record_delimiter`:', + 'value must be a string, a buffer or array of string|buffer,', + `got ${JSON.stringify(options.record_delimiter)}` + ], options); + } + options.record_delimiter = options.record_delimiter.map(function(rd, i){ + if(typeof rd !== 'string' && ! isBuffer$1(rd)){ + throw new CsvError$1('CSV_INVALID_OPTION_RECORD_DELIMITER', [ + 'Invalid option `record_delimiter`:', + 'value must be a string, a buffer or array of string|buffer', + `at index ${i},`, + `got ${JSON.stringify(rd)}` + ], options); + }else if(rd.length === 0){ + throw new CsvError$1('CSV_INVALID_OPTION_RECORD_DELIMITER', [ + 'Invalid option `record_delimiter`:', + 'value must be a non empty string or buffer', + `at index ${i},`, + `got ${JSON.stringify(rd)}` + ], options); } - // Normalize option `skip_empty_lines` - if(typeof options.skip_empty_lines === 'boolean');else if(options.skip_empty_lines === undefined || options.skip_empty_lines === null){ - options.skip_empty_lines = false; - }else { - throw new Error(`Invalid Option: skip_empty_lines must be a boolean, got ${JSON.stringify(options.skip_empty_lines)}`); + if(typeof rd === 'string'){ + rd = Buffer.from(rd, options.encoding); } - // Normalize option `skip_records_with_empty_values` - if(typeof options.skip_records_with_empty_values === 'boolean');else if(options.skip_records_with_empty_values === undefined || options.skip_records_with_empty_values === null){ - options.skip_records_with_empty_values = false; - }else { - throw new Error(`Invalid Option: skip_records_with_empty_values must be a boolean, got ${JSON.stringify(options.skip_records_with_empty_values)}`); + return rd; + }); + // Normalize option `relax_column_count` + if(typeof options.relax_column_count === 'boolean');else if(options.relax_column_count === undefined || options.relax_column_count === null){ + options.relax_column_count = false; + }else { + throw new Error(`Invalid Option: relax_column_count must be a boolean, got ${JSON.stringify(options.relax_column_count)}`); + } + if(typeof options.relax_column_count_less === 'boolean');else if(options.relax_column_count_less === undefined || options.relax_column_count_less === null){ + options.relax_column_count_less = false; + }else { + throw new Error(`Invalid Option: relax_column_count_less must be a boolean, got ${JSON.stringify(options.relax_column_count_less)}`); + } + if(typeof options.relax_column_count_more === 'boolean');else if(options.relax_column_count_more === undefined || options.relax_column_count_more === null){ + options.relax_column_count_more = false; + }else { + throw new Error(`Invalid Option: relax_column_count_more must be a boolean, got ${JSON.stringify(options.relax_column_count_more)}`); + } + // Normalize option `relax_quotes` + if(typeof options.relax_quotes === 'boolean');else if(options.relax_quotes === undefined || options.relax_quotes === null){ + options.relax_quotes = false; + }else { + throw new Error(`Invalid Option: relax_quotes must be a boolean, got ${JSON.stringify(options.relax_quotes)}`); + } + // Normalize option `skip_empty_lines` + if(typeof options.skip_empty_lines === 'boolean');else if(options.skip_empty_lines === undefined || options.skip_empty_lines === null){ + options.skip_empty_lines = false; + }else { + throw new Error(`Invalid Option: skip_empty_lines must be a boolean, got ${JSON.stringify(options.skip_empty_lines)}`); + } + // Normalize option `skip_records_with_empty_values` + if(typeof options.skip_records_with_empty_values === 'boolean');else if(options.skip_records_with_empty_values === undefined || options.skip_records_with_empty_values === null){ + options.skip_records_with_empty_values = false; + }else { + throw new Error(`Invalid Option: skip_records_with_empty_values must be a boolean, got ${JSON.stringify(options.skip_records_with_empty_values)}`); + } + // Normalize option `skip_records_with_error` + if(typeof options.skip_records_with_error === 'boolean');else if(options.skip_records_with_error === undefined || options.skip_records_with_error === null){ + options.skip_records_with_error = false; + }else { + throw new Error(`Invalid Option: skip_records_with_error must be a boolean, got ${JSON.stringify(options.skip_records_with_error)}`); + } + // Normalize option `rtrim` + if(options.rtrim === undefined || options.rtrim === null || options.rtrim === false){ + options.rtrim = false; + }else if(options.rtrim !== true){ + throw new Error(`Invalid Option: rtrim must be a boolean, got ${JSON.stringify(options.rtrim)}`); + } + // Normalize option `ltrim` + if(options.ltrim === undefined || options.ltrim === null || options.ltrim === false){ + options.ltrim = false; + }else if(options.ltrim !== true){ + throw new Error(`Invalid Option: ltrim must be a boolean, got ${JSON.stringify(options.ltrim)}`); + } + // Normalize option `trim` + if(options.trim === undefined || options.trim === null || options.trim === false){ + options.trim = false; + }else if(options.trim !== true){ + throw new Error(`Invalid Option: trim must be a boolean, got ${JSON.stringify(options.trim)}`); + } + // Normalize options `trim`, `ltrim` and `rtrim` + if(options.trim === true && opts.ltrim !== false){ + options.ltrim = true; + }else if(options.ltrim !== true){ + options.ltrim = false; + } + if(options.trim === true && opts.rtrim !== false){ + options.rtrim = true; + }else if(options.rtrim !== true){ + options.rtrim = false; + } + // Normalize option `to` + if(options.to === undefined || options.to === null){ + options.to = -1; + }else { + if(typeof options.to === 'string' && /\d+/.test(options.to)){ + options.to = parseInt(options.to); } - // Normalize option `skip_records_with_error` - if(typeof options.skip_records_with_error === 'boolean');else if(options.skip_records_with_error === undefined || options.skip_records_with_error === null){ - options.skip_records_with_error = false; - }else { - throw new Error(`Invalid Option: skip_records_with_error must be a boolean, got ${JSON.stringify(options.skip_records_with_error)}`); - } - // Normalize option `rtrim` - if(options.rtrim === undefined || options.rtrim === null || options.rtrim === false){ - options.rtrim = false; - }else if(options.rtrim !== true){ - throw new Error(`Invalid Option: rtrim must be a boolean, got ${JSON.stringify(options.rtrim)}`); - } - // Normalize option `ltrim` - if(options.ltrim === undefined || options.ltrim === null || options.ltrim === false){ - options.ltrim = false; - }else if(options.ltrim !== true){ - throw new Error(`Invalid Option: ltrim must be a boolean, got ${JSON.stringify(options.ltrim)}`); - } - // Normalize option `trim` - if(options.trim === undefined || options.trim === null || options.trim === false){ - options.trim = false; - }else if(options.trim !== true){ - throw new Error(`Invalid Option: trim must be a boolean, got ${JSON.stringify(options.trim)}`); - } - // Normalize options `trim`, `ltrim` and `rtrim` - if(options.trim === true && opts.ltrim !== false){ - options.ltrim = true; - }else if(options.ltrim !== true){ - options.ltrim = false; - } - if(options.trim === true && opts.rtrim !== false){ - options.rtrim = true; - }else if(options.rtrim !== true){ - options.rtrim = false; - } - // Normalize option `to` - if(options.to === undefined || options.to === null){ - options.to = -1; - }else { - if(typeof options.to === 'string' && /\d+/.test(options.to)){ - options.to = parseInt(options.to); + if(Number.isInteger(options.to)){ + if(options.to <= 0){ + throw new Error(`Invalid Option: to must be a positive integer greater than 0, got ${JSON.stringify(opts.to)}`); } - if(Number.isInteger(options.to)){ - if(options.to <= 0){ - throw new Error(`Invalid Option: to must be a positive integer greater than 0, got ${JSON.stringify(opts.to)}`); - } - }else { - throw new Error(`Invalid Option: to must be an integer, got ${JSON.stringify(opts.to)}`); - } - } - // Normalize option `to_line` - if(options.to_line === undefined || options.to_line === null){ - options.to_line = -1; }else { - if(typeof options.to_line === 'string' && /\d+/.test(options.to_line)){ - options.to_line = parseInt(options.to_line); - } - if(Number.isInteger(options.to_line)){ - if(options.to_line <= 0){ - throw new Error(`Invalid Option: to_line must be a positive integer greater than 0, got ${JSON.stringify(opts.to_line)}`); - } - }else { - throw new Error(`Invalid Option: to_line must be an integer, got ${JSON.stringify(opts.to_line)}`); - } + throw new Error(`Invalid Option: to must be an integer, got ${JSON.stringify(opts.to)}`); } - this.info = { - bytes: 0, - comment_lines: 0, - empty_lines: 0, - invalid_field_length: 0, - lines: 1, - records: 0 - }; - this.options = options; - this.state = { - bomSkipped: false, - bufBytesStart: 0, - castField: fnCastField, - commenting: false, - // Current error encountered by a record - error: undefined, - enabled: options.from_line === 1, - escaping: false, - escapeIsQuote: isBuffer$1(options.escape) && isBuffer$1(options.quote) && Buffer.compare(options.escape, options.quote) === 0, - // columns can be `false`, `true`, `Array` - expectedRecordLength: Array.isArray(options.columns) ? options.columns.length : undefined, - field: new ResizeableBuffer(20), - firstLineToHeaders: fnFirstLineToHeaders, - needMoreDataSize: Math.max( - // Skip if the remaining buffer smaller than comment - options.comment !== null ? options.comment.length : 0, - // Skip if the remaining buffer can be delimiter - ...options.delimiter.map((delimiter) => delimiter.length), - // Skip if the remaining buffer can be escape sequence - options.quote !== null ? options.quote.length : 0, - ), - previousBuf: undefined, - quoting: false, - stop: false, - rawBuffer: new ResizeableBuffer(100), - record: [], - recordHasError: false, - record_length: 0, - recordDelimiterMaxLength: options.record_delimiter.length === 0 ? 2 : Math.max(...options.record_delimiter.map((v) => v.length)), - trimChars: [Buffer.from(' ', options.encoding)[0], Buffer.from('\t', options.encoding)[0]], - wasQuoting: false, - wasRowDelimiter: false - }; } - // Implementation of `Transform._transform` - _transform(buf, encoding, callback){ - if(this.state.stop === true){ - return; - } - const err = this.__parse(buf, false); - if(err !== undefined){ - this.state.stop = true; - } - callback(err); - } - // Implementation of `Transform._flush` - _flush(callback){ - if(this.state.stop === true){ - return; + // Normalize option `to_line` + if(options.to_line === undefined || options.to_line === null){ + options.to_line = -1; + }else { + if(typeof options.to_line === 'string' && /\d+/.test(options.to_line)){ + options.to_line = parseInt(options.to_line); } - const err = this.__parse(undefined, true); - callback(err); - } - // Central parser implementation - __parse(nextBuf, end){ - const {bom, comment, escape, from_line, ltrim, max_record_size, quote, raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options; - let {record_delimiter} = this.options; - const {bomSkipped, previousBuf, rawBuffer, escapeIsQuote} = this.state; - let buf; - if(previousBuf === undefined){ - if(nextBuf === undefined){ - // Handle empty string - this.push(null); - return; - }else { - buf = nextBuf; + if(Number.isInteger(options.to_line)){ + if(options.to_line <= 0){ + throw new Error(`Invalid Option: to_line must be a positive integer greater than 0, got ${JSON.stringify(opts.to_line)}`); } - }else if(previousBuf !== undefined && nextBuf === undefined){ - buf = previousBuf; }else { - buf = Buffer.concat([previousBuf, nextBuf]); - } - // Handle UTF BOM - if(bomSkipped === false){ - if(bom === false){ - this.state.bomSkipped = true; - }else if(buf.length < 3){ - // No enough data - if(end === false){ - // Wait for more data - this.state.previousBuf = buf; + throw new Error(`Invalid Option: to_line must be an integer, got ${JSON.stringify(opts.to_line)}`); + } + } + return options; + }; + + const isRecordEmpty = function(record){ + return record.every((field) => field == null || field.toString && field.toString().trim() === ''); + }; + + // white space characters + // https://en.wikipedia.org/wiki/Whitespace_character + // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions/Character_Classes#Types + // \f\n\r\t\v\u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff + const tab = 9; + const nl = 10; // \n, 0x0A in hexadecimal, 10 in decimal + const np = 12; + const cr = 13; // \r, 0x0D in hexadécimal, 13 in decimal + const space = 32; + const boms = { + // Note, the following are equals: + // Buffer.from("\ufeff") + // Buffer.from([239, 187, 191]) + // Buffer.from('EFBBBF', 'hex') + 'utf8': Buffer.from([239, 187, 191]), + // Note, the following are equals: + // Buffer.from "\ufeff", 'utf16le + // Buffer.from([255, 254]) + 'utf16le': Buffer.from([255, 254]) + }; + + const transform$1 = function(original_options = {}) { + const info = { + bytes: 0, + comment_lines: 0, + empty_lines: 0, + invalid_field_length: 0, + lines: 1, + records: 0 + }; + const options = normalize_options$1(original_options); + return { + info: info, + original_options: original_options, + options: options, + state: init_state(options), + __needMoreData: function(i, bufLen, end){ + if(end) return false; + const {quote} = this.options; + const {quoting, needMoreDataSize, recordDelimiterMaxLength} = this.state; + const numOfCharLeft = bufLen - i - 1; + const requiredLength = Math.max( + needMoreDataSize, + // Skip if the remaining buffer smaller than record delimiter + recordDelimiterMaxLength, + // Skip if the remaining buffer can be record delimiter following the closing quote + // 1 is for quote.length + quoting ? (quote.length + recordDelimiterMaxLength) : 0, + ); + return numOfCharLeft < requiredLength; + }, + // Central parser implementation + parse: function(nextBuf, end, push, close){ + const {bom, comment, escape, from_line, ltrim, max_record_size, quote, raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options; + let {record_delimiter} = this.options; + const {bomSkipped, previousBuf, rawBuffer, escapeIsQuote} = this.state; + let buf; + if(previousBuf === undefined){ + if(nextBuf === undefined){ + // Handle empty string + close(); return; + }else { + buf = nextBuf; } + }else if(previousBuf !== undefined && nextBuf === undefined){ + buf = previousBuf; }else { - for(const encoding in boms){ - if(boms[encoding].compare(buf, 0, boms[encoding].length) === 0){ - // Skip BOM - const bomLength = boms[encoding].length; - this.state.bufBytesStart += bomLength; - buf = buf.slice(bomLength); - // Renormalize original options with the new encoding - this.__normalizeOptions({...this.__originalOptions, encoding: encoding}); - break; + buf = Buffer.concat([previousBuf, nextBuf]); + } + // Handle UTF BOM + if(bomSkipped === false){ + if(bom === false){ + this.state.bomSkipped = true; + }else if(buf.length < 3){ + // No enough data + if(end === false){ + // Wait for more data + this.state.previousBuf = buf; + return; + } + }else { + for(const encoding in boms){ + if(boms[encoding].compare(buf, 0, boms[encoding].length) === 0){ + // Skip BOM + const bomLength = boms[encoding].length; + this.state.bufBytesStart += bomLength; + buf = buf.slice(bomLength); + // Renormalize original options with the new encoding + this.options = normalize_options$1({...this.original_options, encoding: encoding}); + break; + } } + this.state.bomSkipped = true; } - this.state.bomSkipped = true; } - } - const bufLen = buf.length; - let pos; - for(pos = 0; pos < bufLen; pos++){ - // Ensure we get enough space to look ahead - // There should be a way to move this out of the loop - if(this.__needMoreData(pos, bufLen, end)){ - break; - } - if(this.state.wasRowDelimiter === true){ - this.info.lines++; - this.state.wasRowDelimiter = false; - } - if(to_line !== -1 && this.info.lines > to_line){ - this.state.stop = true; - this.push(null); - return; - } - // Auto discovery of record_delimiter, unix, mac and windows supported - if(this.state.quoting === false && record_delimiter.length === 0){ - const record_delimiterCount = this.__autoDiscoverRecordDelimiter(buf, pos); - if(record_delimiterCount){ - record_delimiter = this.options.record_delimiter; + const bufLen = buf.length; + let pos; + for(pos = 0; pos < bufLen; pos++){ + // Ensure we get enough space to look ahead + // There should be a way to move this out of the loop + if(this.__needMoreData(pos, bufLen, end)){ + break; } - } - const chr = buf[pos]; - if(raw === true){ - rawBuffer.append(chr); - } - if((chr === cr || chr === nl) && this.state.wasRowDelimiter === false){ - this.state.wasRowDelimiter = true; - } - // Previous char was a valid escape char - // treat the current char as a regular char - if(this.state.escaping === true){ - this.state.escaping = false; - }else { - // Escape is only active inside quoted fields - // We are quoting, the char is an escape chr and there is a chr to escape - // if(escape !== null && this.state.quoting === true && chr === escape && pos + 1 < bufLen){ - if(escape !== null && this.state.quoting === true && this.__isEscape(buf, pos, chr) && pos + escape.length < bufLen){ - if(escapeIsQuote){ - if(this.__isQuote(buf, pos+escape.length)){ - this.state.escaping = true; - pos += escape.length - 1; - continue; - } - }else { - this.state.escaping = true; - pos += escape.length - 1; - continue; + if(this.state.wasRowDelimiter === true){ + this.info.lines++; + this.state.wasRowDelimiter = false; + } + if(to_line !== -1 && this.info.lines > to_line){ + this.state.stop = true; + close(); + return; + } + // Auto discovery of record_delimiter, unix, mac and windows supported + if(this.state.quoting === false && record_delimiter.length === 0){ + const record_delimiterCount = this.__autoDiscoverRecordDelimiter(buf, pos); + if(record_delimiterCount){ + record_delimiter = this.options.record_delimiter; } } - // Not currently escaping and chr is a quote - // TODO: need to compare bytes instead of single char - if(this.state.commenting === false && this.__isQuote(buf, pos)){ - if(this.state.quoting === true){ - const nextChr = buf[pos+quote.length]; - const isNextChrTrimable = rtrim && this.__isCharTrimable(nextChr); - const isNextChrComment = comment !== null && this.__compareBytes(comment, buf, pos+quote.length, nextChr); - const isNextChrDelimiter = this.__isDelimiter(buf, pos+quote.length, nextChr); - const isNextChrRecordDelimiter = record_delimiter.length === 0 ? this.__autoDiscoverRecordDelimiter(buf, pos+quote.length) : this.__isRecordDelimiter(nextChr, buf, pos+quote.length); - // Escape a quote - // Treat next char as a regular character - if(escape !== null && this.__isEscape(buf, pos, chr) && this.__isQuote(buf, pos + escape.length)){ + const chr = buf[pos]; + if(raw === true){ + rawBuffer.append(chr); + } + if((chr === cr || chr === nl) && this.state.wasRowDelimiter === false){ + this.state.wasRowDelimiter = true; + } + // Previous char was a valid escape char + // treat the current char as a regular char + if(this.state.escaping === true){ + this.state.escaping = false; + }else { + // Escape is only active inside quoted fields + // We are quoting, the char is an escape chr and there is a chr to escape + // if(escape !== null && this.state.quoting === true && chr === escape && pos + 1 < bufLen){ + if(escape !== null && this.state.quoting === true && this.__isEscape(buf, pos, chr) && pos + escape.length < bufLen){ + if(escapeIsQuote){ + if(this.__isQuote(buf, pos+escape.length)){ + this.state.escaping = true; + pos += escape.length - 1; + continue; + } + }else { + this.state.escaping = true; pos += escape.length - 1; - }else if(!nextChr || isNextChrDelimiter || isNextChrRecordDelimiter || isNextChrComment || isNextChrTrimable){ - this.state.quoting = false; - this.state.wasQuoting = true; - pos += quote.length - 1; continue; - }else if(relax_quotes === false){ - const err = this.__error( - new CsvError$1('CSV_INVALID_CLOSING_QUOTE', [ - 'Invalid Closing Quote:', - `got "${String.fromCharCode(nextChr)}"`, - `at line ${this.info.lines}`, - 'instead of delimiter, record delimiter, trimable character', - '(if activated) or comment', - ], this.options, this.__infoField()) - ); - if(err !== undefined) return err; - }else { - this.state.quoting = false; - this.state.wasQuoting = true; - this.state.field.prepend(quote); - pos += quote.length - 1; } - }else { - if(this.state.field.length !== 0){ - // In relax_quotes mode, treat opening quote preceded by chrs as regular - if(relax_quotes === false){ + } + // Not currently escaping and chr is a quote + // TODO: need to compare bytes instead of single char + if(this.state.commenting === false && this.__isQuote(buf, pos)){ + if(this.state.quoting === true){ + const nextChr = buf[pos+quote.length]; + const isNextChrTrimable = rtrim && this.__isCharTrimable(nextChr); + const isNextChrComment = comment !== null && this.__compareBytes(comment, buf, pos+quote.length, nextChr); + const isNextChrDelimiter = this.__isDelimiter(buf, pos+quote.length, nextChr); + const isNextChrRecordDelimiter = record_delimiter.length === 0 ? this.__autoDiscoverRecordDelimiter(buf, pos+quote.length) : this.__isRecordDelimiter(nextChr, buf, pos+quote.length); + // Escape a quote + // Treat next char as a regular character + if(escape !== null && this.__isEscape(buf, pos, chr) && this.__isQuote(buf, pos + escape.length)){ + pos += escape.length - 1; + }else if(!nextChr || isNextChrDelimiter || isNextChrRecordDelimiter || isNextChrComment || isNextChrTrimable){ + this.state.quoting = false; + this.state.wasQuoting = true; + pos += quote.length - 1; + continue; + }else if(relax_quotes === false){ const err = this.__error( - new CsvError$1('INVALID_OPENING_QUOTE', [ - 'Invalid Opening Quote:', - `a quote is found inside a field at line ${this.info.lines}`, - ], this.options, this.__infoField(), { - field: this.state.field, - }) + new CsvError$1('CSV_INVALID_CLOSING_QUOTE', [ + 'Invalid Closing Quote:', + `got "${String.fromCharCode(nextChr)}"`, + `at line ${this.info.lines}`, + 'instead of delimiter, record delimiter, trimable character', + '(if activated) or comment', + ], this.options, this.__infoField()) ); if(err !== undefined) return err; + }else { + this.state.quoting = false; + this.state.wasQuoting = true; + this.state.field.prepend(quote); + pos += quote.length - 1; } }else { - this.state.quoting = true; - pos += quote.length - 1; - continue; - } - } - } - if(this.state.quoting === false){ - const recordDelimiterLength = this.__isRecordDelimiter(chr, buf, pos); - if(recordDelimiterLength !== 0){ - // Do not emit comments which take a full line - const skipCommentLine = this.state.commenting && (this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0); - if(skipCommentLine){ - this.info.comment_lines++; - // Skip full comment line - }else { - // Activate records emition if above from_line - if(this.state.enabled === false && this.info.lines + (this.state.wasRowDelimiter === true ? 1: 0) >= from_line){ - this.state.enabled = true; - this.__resetField(); - this.__resetRecord(); - pos += recordDelimiterLength - 1; + if(this.state.field.length !== 0){ + // In relax_quotes mode, treat opening quote preceded by chrs as regular + if(relax_quotes === false){ + const err = this.__error( + new CsvError$1('INVALID_OPENING_QUOTE', [ + 'Invalid Opening Quote:', + `a quote is found inside a field at line ${this.info.lines}`, + ], this.options, this.__infoField(), { + field: this.state.field, + }) + ); + if(err !== undefined) return err; + } + }else { + this.state.quoting = true; + pos += quote.length - 1; continue; } - // Skip if line is empty and skip_empty_lines activated - if(skip_empty_lines === true && this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0){ - this.info.empty_lines++; - pos += recordDelimiterLength - 1; - continue; + } + } + if(this.state.quoting === false){ + const recordDelimiterLength = this.__isRecordDelimiter(chr, buf, pos); + if(recordDelimiterLength !== 0){ + // Do not emit comments which take a full line + const skipCommentLine = this.state.commenting && (this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0); + if(skipCommentLine){ + this.info.comment_lines++; + // Skip full comment line + }else { + // Activate records emition if above from_line + if(this.state.enabled === false && this.info.lines + (this.state.wasRowDelimiter === true ? 1: 0) >= from_line){ + this.state.enabled = true; + this.__resetField(); + this.__resetRecord(); + pos += recordDelimiterLength - 1; + continue; + } + // Skip if line is empty and skip_empty_lines activated + if(skip_empty_lines === true && this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0){ + this.info.empty_lines++; + pos += recordDelimiterLength - 1; + continue; + } + this.info.bytes = this.state.bufBytesStart + pos; + const errField = this.__onField(); + if(errField !== undefined) return errField; + this.info.bytes = this.state.bufBytesStart + pos + recordDelimiterLength; + const errRecord = this.__onRecord(push); + if(errRecord !== undefined) return errRecord; + if(to !== -1 && this.info.records >= to){ + this.state.stop = true; + close(); + return; + } } + this.state.commenting = false; + pos += recordDelimiterLength - 1; + continue; + } + if(this.state.commenting){ + continue; + } + const commentCount = comment === null ? 0 : this.__compareBytes(comment, buf, pos, chr); + if(commentCount !== 0){ + this.state.commenting = true; + continue; + } + const delimiterLength = this.__isDelimiter(buf, pos, chr); + if(delimiterLength !== 0){ this.info.bytes = this.state.bufBytesStart + pos; const errField = this.__onField(); if(errField !== undefined) return errField; - this.info.bytes = this.state.bufBytesStart + pos + recordDelimiterLength; - const errRecord = this.__onRecord(); - if(errRecord !== undefined) return errRecord; - if(to !== -1 && this.info.records >= to){ - this.state.stop = true; - this.push(null); - return; - } + pos += delimiterLength - 1; + continue; } - this.state.commenting = false; - pos += recordDelimiterLength - 1; - continue; } - if(this.state.commenting){ - continue; - } - const commentCount = comment === null ? 0 : this.__compareBytes(comment, buf, pos, chr); - if(commentCount !== 0){ - this.state.commenting = true; - continue; - } - const delimiterLength = this.__isDelimiter(buf, pos, chr); - if(delimiterLength !== 0){ - this.info.bytes = this.state.bufBytesStart + pos; - const errField = this.__onField(); - if(errField !== undefined) return errField; - pos += delimiterLength - 1; - continue; + } + if(this.state.commenting === false){ + if(max_record_size !== 0 && this.state.record_length + this.state.field.length > max_record_size){ + const err = this.__error( + new CsvError$1('CSV_MAX_RECORD_SIZE', [ + 'Max Record Size:', + 'record exceed the maximum number of tolerated bytes', + `of ${max_record_size}`, + `at line ${this.info.lines}`, + ], this.options, this.__infoField()) + ); + if(err !== undefined) return err; } } - } - if(this.state.commenting === false){ - if(max_record_size !== 0 && this.state.record_length + this.state.field.length > max_record_size){ + const lappend = ltrim === false || this.state.quoting === true || this.state.field.length !== 0 || !this.__isCharTrimable(chr); + // rtrim in non quoting is handle in __onField + const rappend = rtrim === false || this.state.wasQuoting === false; + if(lappend === true && rappend === true){ + this.state.field.append(chr); + }else if(rtrim === true && !this.__isCharTrimable(chr)){ const err = this.__error( - new CsvError$1('CSV_MAX_RECORD_SIZE', [ - 'Max Record Size:', - 'record exceed the maximum number of tolerated bytes', - `of ${max_record_size}`, + new CsvError$1('CSV_NON_TRIMABLE_CHAR_AFTER_CLOSING_QUOTE', [ + 'Invalid Closing Quote:', + 'found non trimable byte after quote', `at line ${this.info.lines}`, ], this.options, this.__infoField()) ); if(err !== undefined) return err; } } - const lappend = ltrim === false || this.state.quoting === true || this.state.field.length !== 0 || !this.__isCharTrimable(chr); - // rtrim in non quoting is handle in __onField - const rappend = rtrim === false || this.state.wasQuoting === false; - if(lappend === true && rappend === true){ - this.state.field.append(chr); - }else if(rtrim === true && !this.__isCharTrimable(chr)){ - const err = this.__error( - new CsvError$1('CSV_NON_TRIMABLE_CHAR_AFTER_CLOSING_QUOTE', [ - 'Invalid Closing Quote:', - 'found non trimable byte after quote', - `at line ${this.info.lines}`, - ], this.options, this.__infoField()) - ); - if(err !== undefined) return err; - } - } - if(end === true){ - // Ensure we are not ending in a quoting state - if(this.state.quoting === true){ - const err = this.__error( - new CsvError$1('CSV_QUOTE_NOT_CLOSED', [ - 'Quote Not Closed:', - `the parsing is finished with an opening quote at line ${this.info.lines}`, - ], this.options, this.__infoField()) - ); - if(err !== undefined) return err; + if(end === true){ + // Ensure we are not ending in a quoting state + if(this.state.quoting === true){ + const err = this.__error( + new CsvError$1('CSV_QUOTE_NOT_CLOSED', [ + 'Quote Not Closed:', + `the parsing is finished with an opening quote at line ${this.info.lines}`, + ], this.options, this.__infoField()) + ); + if(err !== undefined) return err; + }else { + // Skip last line if it has no characters + if(this.state.wasQuoting === true || this.state.record.length !== 0 || this.state.field.length !== 0){ + this.info.bytes = this.state.bufBytesStart + pos; + const errField = this.__onField(); + if(errField !== undefined) return errField; + const errRecord = this.__onRecord(push); + if(errRecord !== undefined) return errRecord; + }else if(this.state.wasRowDelimiter === true){ + this.info.empty_lines++; + }else if(this.state.commenting === true){ + this.info.comment_lines++; + } + } }else { - // Skip last line if it has no characters - if(this.state.wasQuoting === true || this.state.record.length !== 0 || this.state.field.length !== 0){ - this.info.bytes = this.state.bufBytesStart + pos; - const errField = this.__onField(); - if(errField !== undefined) return errField; - const errRecord = this.__onRecord(); - if(errRecord !== undefined) return errRecord; - }else if(this.state.wasRowDelimiter === true){ - this.info.empty_lines++; - }else if(this.state.commenting === true){ - this.info.comment_lines++; + this.state.bufBytesStart += pos; + this.state.previousBuf = buf.slice(pos); + } + if(this.state.wasRowDelimiter === true){ + this.info.lines++; + this.state.wasRowDelimiter = false; + } + }, + __onRecord: function(push){ + const {columns, group_columns_by_name, encoding, info, from, relax_column_count, relax_column_count_less, relax_column_count_more, raw, skip_records_with_empty_values} = this.options; + const {enabled, record} = this.state; + if(enabled === false){ + return this.__resetRecord(); + } + // Convert the first line into column names + const recordLength = record.length; + if(columns === true){ + if(skip_records_with_empty_values === true && isRecordEmpty(record)){ + this.__resetRecord(); + return; + } + return this.__firstLineToColumns(record); + } + if(columns === false && this.info.records === 0){ + this.state.expectedRecordLength = recordLength; + } + if(recordLength !== this.state.expectedRecordLength){ + const err = columns === false ? + new CsvError$1('CSV_RECORD_INCONSISTENT_FIELDS_LENGTH', [ + 'Invalid Record Length:', + `expect ${this.state.expectedRecordLength},`, + `got ${recordLength} on line ${this.info.lines}`, + ], this.options, this.__infoField(), { + record: record, + }) + : + new CsvError$1('CSV_RECORD_INCONSISTENT_COLUMNS', [ + 'Invalid Record Length:', + `columns length is ${columns.length},`, // rename columns + `got ${recordLength} on line ${this.info.lines}`, + ], this.options, this.__infoField(), { + record: record, + }); + if(relax_column_count === true || + (relax_column_count_less === true && recordLength < this.state.expectedRecordLength) || + (relax_column_count_more === true && recordLength > this.state.expectedRecordLength)){ + this.info.invalid_field_length++; + this.state.error = err; + // Error is undefined with skip_records_with_error + }else { + const finalErr = this.__error(err); + if(finalErr) return finalErr; } } - }else { - this.state.bufBytesStart += pos; - this.state.previousBuf = buf.slice(pos); - } - if(this.state.wasRowDelimiter === true){ - this.info.lines++; - this.state.wasRowDelimiter = false; - } - } - __onRecord(){ - const {columns, group_columns_by_name, encoding, info, from, relax_column_count, relax_column_count_less, relax_column_count_more, raw, skip_records_with_empty_values} = this.options; - const {enabled, record} = this.state; - if(enabled === false){ - return this.__resetRecord(); - } - // Convert the first line into column names - const recordLength = record.length; - if(columns === true){ if(skip_records_with_empty_values === true && isRecordEmpty(record)){ this.__resetRecord(); return; } - return this.__firstLineToColumns(record); - } - if(columns === false && this.info.records === 0){ - this.state.expectedRecordLength = recordLength; - } - if(recordLength !== this.state.expectedRecordLength){ - const err = columns === false ? - new CsvError$1('CSV_RECORD_INCONSISTENT_FIELDS_LENGTH', [ - 'Invalid Record Length:', - `expect ${this.state.expectedRecordLength},`, - `got ${recordLength} on line ${this.info.lines}`, - ], this.options, this.__infoField(), { - record: record, - }) - : - new CsvError$1('CSV_RECORD_INCONSISTENT_COLUMNS', [ - 'Invalid Record Length:', - `columns length is ${columns.length},`, // rename columns - `got ${recordLength} on line ${this.info.lines}`, - ], this.options, this.__infoField(), { - record: record, - }); - if(relax_column_count === true || - (relax_column_count_less === true && recordLength < this.state.expectedRecordLength) || - (relax_column_count_more === true && recordLength > this.state.expectedRecordLength)){ - this.info.invalid_field_length++; - this.state.error = err; - // Error is undefined with skip_records_with_error - }else { - const finalErr = this.__error(err); - if(finalErr) return finalErr; + if(this.state.recordHasError === true){ + this.__resetRecord(); + this.state.recordHasError = false; + return; } - } - if(skip_records_with_empty_values === true && isRecordEmpty(record)){ - this.__resetRecord(); - return; - } - if(this.state.recordHasError === true){ - this.__resetRecord(); - this.state.recordHasError = false; - return; - } - this.info.records++; - if(from === 1 || this.info.records >= from){ - const {objname} = this.options; - // With columns, records are object - if(columns !== false){ - const obj = {}; - // Transform record array to an object - for(let i = 0, l = record.length; i < l; i++){ - if(columns[i] === undefined || columns[i].disabled) continue; - // Turn duplicate columns into an array - if (group_columns_by_name === true && obj[columns[i].name] !== undefined) { - if (Array.isArray(obj[columns[i].name])) { - obj[columns[i].name] = obj[columns[i].name].concat(record[i]); + this.info.records++; + if(from === 1 || this.info.records >= from){ + const {objname} = this.options; + // With columns, records are object + if(columns !== false){ + const obj = {}; + // Transform record array to an object + for(let i = 0, l = record.length; i < l; i++){ + if(columns[i] === undefined || columns[i].disabled) continue; + // Turn duplicate columns into an array + if (group_columns_by_name === true && obj[columns[i].name] !== undefined) { + if (Array.isArray(obj[columns[i].name])) { + obj[columns[i].name] = obj[columns[i].name].concat(record[i]); + } else { + obj[columns[i].name] = [obj[columns[i].name], record[i]]; + } } else { - obj[columns[i].name] = [obj[columns[i].name], record[i]]; + obj[columns[i].name] = record[i]; } - } else { - obj[columns[i].name] = record[i]; } - } - // Without objname (default) - if(raw === true || info === true){ - const extRecord = Object.assign( - {record: obj}, - (raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}), - (info === true ? {info: this.__infoRecord()}: {}) - ); - const err = this.__push( - objname === undefined ? extRecord : [obj[objname], extRecord] - ); - if(err){ - return err; - } - }else { - const err = this.__push( - objname === undefined ? obj : [obj[objname], obj] - ); - if(err){ - return err; - } - } - // Without columns, records are array - }else { - if(raw === true || info === true){ - const extRecord = Object.assign( - {record: record}, - raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}, - info === true ? {info: this.__infoRecord()}: {} - ); - const err = this.__push( - objname === undefined ? extRecord : [record[objname], extRecord] - ); - if(err){ - return err; + // Without objname (default) + if(raw === true || info === true){ + const extRecord = Object.assign( + {record: obj}, + (raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}), + (info === true ? {info: this.__infoRecord()}: {}) + ); + const err = this.__push( + objname === undefined ? extRecord : [obj[objname], extRecord] + , push); + if(err){ + return err; + } + }else { + const err = this.__push( + objname === undefined ? obj : [obj[objname], obj] + , push); + if(err){ + return err; + } } + // Without columns, records are array }else { - const err = this.__push( - objname === undefined ? record : [record[objname], record] - ); - if(err){ - return err; + if(raw === true || info === true){ + const extRecord = Object.assign( + {record: record}, + raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}, + info === true ? {info: this.__infoRecord()}: {} + ); + const err = this.__push( + objname === undefined ? extRecord : [record[objname], extRecord] + , push); + if(err){ + return err; + } + }else { + const err = this.__push( + objname === undefined ? record : [record[objname], record] + , push); + if(err){ + return err; + } } } } - } - this.__resetRecord(); - } - __firstLineToColumns(record){ - const {firstLineToHeaders} = this.state; - try{ - const headers = firstLineToHeaders === undefined ? record : firstLineToHeaders.call(null, record); - if(!Array.isArray(headers)){ - return this.__error( - new CsvError$1('CSV_INVALID_COLUMN_MAPPING', [ - 'Invalid Column Mapping:', - 'expect an array from column function,', - `got ${JSON.stringify(headers)}` - ], this.options, this.__infoField(), { - headers: headers, - }) - ); - } - const normalizedHeaders = normalizeColumnsArray(headers); - this.state.expectedRecordLength = normalizedHeaders.length; - this.options.columns = normalizedHeaders; this.__resetRecord(); - return; - }catch(err){ - return err; - } - } - __resetRecord(){ - if(this.options.raw === true){ - this.state.rawBuffer.reset(); - } - this.state.error = undefined; - this.state.record = []; - this.state.record_length = 0; - } - __onField(){ - const {cast, encoding, rtrim, max_record_size} = this.options; - const {enabled, wasQuoting} = this.state; - // Short circuit for the from_line options - if(enabled === false){ - return this.__resetField(); - } - let field = this.state.field.toString(encoding); - if(rtrim === true && wasQuoting === false){ - field = field.trimRight(); - } - if(cast === true){ - const [err, f] = this.__cast(field); - if(err !== undefined) return err; - field = f; - } - this.state.record.push(field); - // Increment record length if record size must not exceed a limit - if(max_record_size !== 0 && typeof field === 'string'){ - this.state.record_length += field.length; - } - this.__resetField(); - } - __resetField(){ - this.state.field.reset(); - this.state.wasQuoting = false; - } - __push(record){ - const {on_record} = this.options; - if(on_record !== undefined){ - const info = this.__infoRecord(); + }, + __firstLineToColumns: function(record){ + const {firstLineToHeaders} = this.state; try{ - record = on_record.call(null, record, info); + const headers = firstLineToHeaders === undefined ? record : firstLineToHeaders.call(null, record); + if(!Array.isArray(headers)){ + return this.__error( + new CsvError$1('CSV_INVALID_COLUMN_MAPPING', [ + 'Invalid Column Mapping:', + 'expect an array from column function,', + `got ${JSON.stringify(headers)}` + ], this.options, this.__infoField(), { + headers: headers, + }) + ); + } + const normalizedHeaders = normalize_columns_array(headers); + this.state.expectedRecordLength = normalizedHeaders.length; + this.options.columns = normalizedHeaders; + this.__resetRecord(); + return; }catch(err){ return err; } - if(record === undefined || record === null){ return; } - } - this.push(record); - } - // Return a tuple with the error and the casted value - __cast(field){ - const {columns, relax_column_count} = this.options; - const isColumns = Array.isArray(columns); - // Dont loose time calling cast - // because the final record is an object - // and this field can't be associated to a key present in columns - if(isColumns === true && relax_column_count && this.options.columns.length <= this.state.record.length){ - return [undefined, undefined]; - } - if(this.state.castField !== null){ - try{ + }, + __resetRecord: function(){ + if(this.options.raw === true){ + this.state.rawBuffer.reset(); + } + this.state.error = undefined; + this.state.record = []; + this.state.record_length = 0; + }, + __onField: function(){ + const {cast, encoding, rtrim, max_record_size} = this.options; + const {enabled, wasQuoting} = this.state; + // Short circuit for the from_line options + if(enabled === false){ + return this.__resetField(); + } + let field = this.state.field.toString(encoding); + if(rtrim === true && wasQuoting === false){ + field = field.trimRight(); + } + if(cast === true){ + const [err, f] = this.__cast(field); + if(err !== undefined) return err; + field = f; + } + this.state.record.push(field); + // Increment record length if record size must not exceed a limit + if(max_record_size !== 0 && typeof field === 'string'){ + this.state.record_length += field.length; + } + this.__resetField(); + }, + __resetField: function(){ + this.state.field.reset(); + this.state.wasQuoting = false; + }, + __push: function(record, push){ + const {on_record} = this.options; + if(on_record !== undefined){ + const info = this.__infoRecord(); + try{ + record = on_record.call(null, record, info); + }catch(err){ + return err; + } + if(record === undefined || record === null){ return; } + } + push(record); + }, + // Return a tuple with the error and the casted value + __cast: function(field){ + const {columns, relax_column_count} = this.options; + const isColumns = Array.isArray(columns); + // Dont loose time calling cast + // because the final record is an object + // and this field can't be associated to a key present in columns + if(isColumns === true && relax_column_count && this.options.columns.length <= this.state.record.length){ + return [undefined, undefined]; + } + if(this.state.castField !== null){ + try{ + const info = this.__infoField(); + return [undefined, this.state.castField.call(null, field, info)]; + }catch(err){ + return [err]; + } + } + if(this.__isFloat(field)){ + return [undefined, parseFloat(field)]; + }else if(this.options.cast_date !== false){ const info = this.__infoField(); - return [undefined, this.state.castField.call(null, field, info)]; - }catch(err){ - return [err]; + return [undefined, this.options.cast_date.call(null, field, info)]; } - } - if(this.__isFloat(field)){ - return [undefined, parseFloat(field)]; - }else if(this.options.cast_date !== false){ - const info = this.__infoField(); - return [undefined, this.options.cast_date.call(null, field, info)]; - } - return [undefined, field]; - } - // Helper to test if a character is a space or a line delimiter - __isCharTrimable(chr){ - return chr === space || chr === tab || chr === cr || chr === nl || chr === np; - } - // Keep it in case we implement the `cast_int` option - // __isInt(value){ - // // return Number.isInteger(parseInt(value)) - // // return !isNaN( parseInt( obj ) ); - // return /^(\-|\+)?[1-9][0-9]*$/.test(value) - // } - __isFloat(value){ - return (value - parseFloat(value) + 1) >= 0; // Borrowed from jquery - } - __compareBytes(sourceBuf, targetBuf, targetPos, firstByte){ - if(sourceBuf[0] !== firstByte) return 0; - const sourceLength = sourceBuf.length; - for(let i = 1; i < sourceLength; i++){ - if(sourceBuf[i] !== targetBuf[targetPos+i]) return 0; - } - return sourceLength; - } - __needMoreData(i, bufLen, end){ - if(end) return false; - const {quote} = this.options; - const {quoting, needMoreDataSize, recordDelimiterMaxLength} = this.state; - const numOfCharLeft = bufLen - i - 1; - const requiredLength = Math.max( - needMoreDataSize, - // Skip if the remaining buffer smaller than record delimiter - recordDelimiterMaxLength, - // Skip if the remaining buffer can be record delimiter following the closing quote - // 1 is for quote.length - quoting ? (quote.length + recordDelimiterMaxLength) : 0, - ); - return numOfCharLeft < requiredLength; - } - __isDelimiter(buf, pos, chr){ - const {delimiter, ignore_last_delimiters} = this.options; - if(ignore_last_delimiters === true && this.state.record.length === this.options.columns.length - 1){ - return 0; - }else if(ignore_last_delimiters !== false && typeof ignore_last_delimiters === 'number' && this.state.record.length === ignore_last_delimiters - 1){ - return 0; - } - loop1: for(let i = 0; i < delimiter.length; i++){ - const del = delimiter[i]; - if(del[0] === chr){ - for(let j = 1; j < del.length; j++){ - if(del[j] !== buf[pos+j]) continue loop1; + return [undefined, field]; + }, + // Helper to test if a character is a space or a line delimiter + __isCharTrimable: function(chr){ + return chr === space || chr === tab || chr === cr || chr === nl || chr === np; + }, + // Keep it in case we implement the `cast_int` option + // __isInt(value){ + // // return Number.isInteger(parseInt(value)) + // // return !isNaN( parseInt( obj ) ); + // return /^(\-|\+)?[1-9][0-9]*$/.test(value) + // } + __isFloat: function(value){ + return (value - parseFloat(value) + 1) >= 0; // Borrowed from jquery + }, + __compareBytes: function(sourceBuf, targetBuf, targetPos, firstByte){ + if(sourceBuf[0] !== firstByte) return 0; + const sourceLength = sourceBuf.length; + for(let i = 1; i < sourceLength; i++){ + if(sourceBuf[i] !== targetBuf[targetPos+i]) return 0; + } + return sourceLength; + }, + __isDelimiter: function(buf, pos, chr){ + const {delimiter, ignore_last_delimiters} = this.options; + if(ignore_last_delimiters === true && this.state.record.length === this.options.columns.length - 1){ + return 0; + }else if(ignore_last_delimiters !== false && typeof ignore_last_delimiters === 'number' && this.state.record.length === ignore_last_delimiters - 1){ + return 0; + } + loop1: for(let i = 0; i < delimiter.length; i++){ + const del = delimiter[i]; + if(del[0] === chr){ + for(let j = 1; j < del.length; j++){ + if(del[j] !== buf[pos+j]) continue loop1; + } + return del.length; } - return del.length; } - } - return 0; - } - __isRecordDelimiter(chr, buf, pos){ - const {record_delimiter} = this.options; - const recordDelimiterLength = record_delimiter.length; - loop1: for(let i = 0; i < recordDelimiterLength; i++){ - const rd = record_delimiter[i]; - const rdLength = rd.length; - if(rd[0] !== chr){ - continue; + return 0; + }, + __isRecordDelimiter: function(chr, buf, pos){ + const {record_delimiter} = this.options; + const recordDelimiterLength = record_delimiter.length; + loop1: for(let i = 0; i < recordDelimiterLength; i++){ + const rd = record_delimiter[i]; + const rdLength = rd.length; + if(rd[0] !== chr){ + continue; + } + for(let j = 1; j < rdLength; j++){ + if(rd[j] !== buf[pos+j]){ + continue loop1; + } + } + return rd.length; } - for(let j = 1; j < rdLength; j++){ - if(rd[j] !== buf[pos+j]){ - continue loop1; + return 0; + }, + __isEscape: function(buf, pos, chr){ + const {escape} = this.options; + if(escape === null) return false; + const l = escape.length; + if(escape[0] === chr){ + for(let i = 0; i < l; i++){ + if(escape[i] !== buf[pos+i]){ + return false; + } } + return true; } - return rd.length; - } - return 0; - } - __isEscape(buf, pos, chr){ - const {escape} = this.options; - if(escape === null) return false; - const l = escape.length; - if(escape[0] === chr){ + return false; + }, + __isQuote: function(buf, pos){ + const {quote} = this.options; + if(quote === null) return false; + const l = quote.length; for(let i = 0; i < l; i++){ - if(escape[i] !== buf[pos+i]){ + if(quote[i] !== buf[pos+i]){ return false; } } return true; - } - return false; - } - __isQuote(buf, pos){ - const {quote} = this.options; - if(quote === null) return false; - const l = quote.length; - for(let i = 0; i < l; i++){ - if(quote[i] !== buf[pos+i]){ - return false; - } - } - return true; - } - __autoDiscoverRecordDelimiter(buf, pos){ - const {encoding} = this.options; - const chr = buf[pos]; - if(chr === cr){ - if(buf[pos+1] === nl){ - this.options.record_delimiter.push(Buffer.from('\r\n', encoding)); - this.state.recordDelimiterMaxLength = 2; - return 2; - }else { - this.options.record_delimiter.push(Buffer.from('\r', encoding)); + }, + __autoDiscoverRecordDelimiter: function(buf, pos){ + const {encoding} = this.options; + const chr = buf[pos]; + if(chr === cr){ + if(buf[pos+1] === nl){ + this.options.record_delimiter.push(Buffer.from('\r\n', encoding)); + this.state.recordDelimiterMaxLength = 2; + return 2; + }else { + this.options.record_delimiter.push(Buffer.from('\r', encoding)); + this.state.recordDelimiterMaxLength = 1; + return 1; + } + }else if(chr === nl){ + this.options.record_delimiter.push(Buffer.from('\n', encoding)); this.state.recordDelimiterMaxLength = 1; return 1; } - }else if(chr === nl){ - this.options.record_delimiter.push(Buffer.from('\n', encoding)); - this.state.recordDelimiterMaxLength = 1; - return 1; - } - return 0; - } - __error(msg){ - const {encoding, raw, skip_records_with_error} = this.options; - const err = typeof msg === 'string' ? new Error(msg) : msg; - if(skip_records_with_error){ - this.state.recordHasError = true; - this.emit('skip', err, raw ? this.state.rawBuffer.toString(encoding) : undefined); - return undefined; - }else { - return err; + return 0; + }, + __error: function(msg){ + const {encoding, raw, skip_records_with_error} = this.options; + const err = typeof msg === 'string' ? new Error(msg) : msg; + if(skip_records_with_error){ + this.state.recordHasError = true; + if(this.options.on_skip !== undefined){ + this.options.on_skip(err, raw ? this.state.rawBuffer.toString(encoding) : undefined); + } + // this.emit('skip', err, raw ? this.state.rawBuffer.toString(encoding) : undefined); + return undefined; + }else { + return err; + } + }, + __infoDataSet: function(){ + return { + ...this.info, + columns: this.options.columns + }; + }, + __infoRecord: function(){ + const {columns, raw, encoding} = this.options; + return { + ...this.__infoDataSet(), + error: this.state.error, + header: columns === true, + index: this.state.record.length, + raw: raw ? this.state.rawBuffer.toString(encoding) : undefined + }; + }, + __infoField: function(){ + const {columns} = this.options; + const isColumns = Array.isArray(columns); + return { + ...this.__infoRecord(), + column: isColumns === true ? + (columns.length > this.state.record.length ? + columns[this.state.record.length].name : + null + ) : + this.state.record.length, + quoting: this.state.wasQuoting, + }; } - } - __infoDataSet(){ - return { - ...this.info, - columns: this.options.columns + }; + }; + + class Parser extends Transform { + constructor(opts = {}){ + super({...{readableObjectMode: true}, ...opts, encoding: null}); + this.api = transform$1(opts); + this.api.options.on_skip = (err, chunk) => { + this.emit('skip', err, chunk); }; + // Backward compatibility + this.state = this.api.state; + this.options = this.api.options; + this.info = this.api.info; } - __infoRecord(){ - const {columns, raw, encoding} = this.options; - return { - ...this.__infoDataSet(), - error: this.state.error, - header: columns === true, - index: this.state.record.length, - raw: raw ? this.state.rawBuffer.toString(encoding) : undefined - }; + // Implementation of `Transform._transform` + _transform(buf, encoding, callback){ + if(this.state.stop === true){ + return; + } + const err = this.api.parse(buf, false, (record) => { + this.push.call(this, record); + }, () => { + this.push.call(this, null); + }); + if(err !== undefined){ + this.state.stop = true; + } + callback(err); } - __infoField(){ - const {columns} = this.options; - const isColumns = Array.isArray(columns); - return { - ...this.__infoRecord(), - column: isColumns === true ? - (columns.length > this.state.record.length ? - columns[this.state.record.length].name : - null - ) : - this.state.record.length, - quoting: this.state.wasQuoting, - }; + // Implementation of `Transform._flush` + _flush(callback){ + if(this.state.stop === true){ + return; + } + const err = this.api.parse(undefined, true, (record) => { + this.push.call(this, record); + }, () => { + this.push.call(this, null); + }); + callback(err); } } @@ -6537,7 +6601,7 @@ const type = typeof argument; if(data === undefined && (typeof argument === 'string' || isBuffer$1(argument))){ data = argument; - }else if(options === undefined && isObject$1(argument)){ + }else if(options === undefined && is_object$1(argument)){ options = argument; }else if(callback === undefined && type === 'function'){ callback = argument; @@ -6562,10 +6626,10 @@ } }); parser.on('error', function(err){ - callback(err, undefined, parser.__infoDataSet()); + callback(err, undefined, parser.api.__infoDataSet()); }); parser.on('end', function(){ - callback(undefined, records, parser.__infoDataSet()); + callback(undefined, records, parser.api.__infoDataSet()); }); } if(data !== undefined){ @@ -6728,8 +6792,6 @@ return transformer; }; - const bom_utf8 = Buffer.from([239, 187, 191]); - class CsvError extends Error { constructor(code, message, ...contexts) { if(Array.isArray(message)) message = message.join(' '); @@ -6747,16 +6809,10 @@ } } - const isObject = function(obj){ + const is_object = function(obj){ return typeof obj === 'object' && obj !== null && ! Array.isArray(obj); }; - const underscore = function(str){ - return str.replace(/([A-Z])/g, function(_, match){ - return '_' + match.toLowerCase(); - }); - }; - // Lodash implementation of `get` const charCodeOfDot = '.'.charCodeAt(0); @@ -6834,441 +6890,472 @@ return (index && index === length) ? object : undefined; }; - class Stringifier extends Transform { - constructor(opts = {}){ - super({...{writableObjectMode: true}, ...opts}); - const options = {}; - let err; - // Merge with user options - for(const opt in opts){ - options[underscore(opt)] = opts[opt]; - } - if((err = this.normalize(options)) !== undefined) throw err; - switch(options.record_delimiter){ - case 'auto': - options.record_delimiter = null; - break; - case 'unix': - options.record_delimiter = "\n"; - break; - case 'mac': - options.record_delimiter = "\r"; - break; - case 'windows': - options.record_delimiter = "\r\n"; - break; - case 'ascii': - options.record_delimiter = "\u001e"; - break; - case 'unicode': - options.record_delimiter = "\u2028"; - break; - } - // Expose options - this.options = options; - // Internal state - this.state = { - stop: false - }; - // Information - this.info = { - records: 0 - }; + const normalize_columns = function(columns){ + if(columns === undefined || columns === null){ + return [undefined, undefined]; } - normalize(options){ - // Normalize option `bom` - if(options.bom === undefined || options.bom === null || options.bom === false){ - options.bom = false; - }else if(options.bom !== true){ - return new CsvError('CSV_OPTION_BOOLEAN_INVALID_TYPE', [ - 'option `bom` is optional and must be a boolean value,', - `got ${JSON.stringify(options.bom)}` - ]); - } - // Normalize option `delimiter` - if(options.delimiter === undefined || options.delimiter === null){ - options.delimiter = ','; - }else if(isBuffer$1(options.delimiter)){ - options.delimiter = options.delimiter.toString(); - }else if(typeof options.delimiter !== 'string'){ - return new CsvError('CSV_OPTION_DELIMITER_INVALID_TYPE', [ - 'option `delimiter` must be a buffer or a string,', - `got ${JSON.stringify(options.delimiter)}` - ]); - } - // Normalize option `quote` - if(options.quote === undefined || options.quote === null){ - options.quote = '"'; - }else if(options.quote === true){ - options.quote = '"'; - }else if(options.quote === false){ - options.quote = ''; - }else if (isBuffer$1(options.quote)){ - options.quote = options.quote.toString(); - }else if(typeof options.quote !== 'string'){ - return new CsvError('CSV_OPTION_QUOTE_INVALID_TYPE', [ - 'option `quote` must be a boolean, a buffer or a string,', - `got ${JSON.stringify(options.quote)}` - ]); + if(typeof columns !== 'object'){ + return [Error('Invalid option "columns": expect an array or an object')]; + } + if(!Array.isArray(columns)){ + const newcolumns = []; + for(const k in columns){ + newcolumns.push({ + key: k, + header: columns[k] + }); } - // Normalize option `quoted` - if(options.quoted === undefined || options.quoted === null){ - options.quoted = false; - } - // Normalize option `quoted_empty` - if(options.quoted_empty === undefined || options.quoted_empty === null){ - options.quoted_empty = undefined; - } - // Normalize option `quoted_match` - if(options.quoted_match === undefined || options.quoted_match === null || options.quoted_match === false){ - options.quoted_match = null; - }else if(!Array.isArray(options.quoted_match)){ - options.quoted_match = [options.quoted_match]; - } - if(options.quoted_match){ - for(const quoted_match of options.quoted_match){ - const isString = typeof quoted_match === 'string'; - const isRegExp = quoted_match instanceof RegExp; - if(!isString && !isRegExp){ - return Error(`Invalid Option: quoted_match must be a string or a regex, got ${JSON.stringify(quoted_match)}`); + columns = newcolumns; + }else { + const newcolumns = []; + for(const column of columns){ + if(typeof column === 'string'){ + newcolumns.push({ + key: column, + header: column + }); + }else if(typeof column === 'object' && column !== null && !Array.isArray(column)){ + if(!column.key){ + return [Error('Invalid column definition: property "key" is required')]; } + if(column.header === undefined){ + column.header = column.key; + } + newcolumns.push(column); + }else { + return [Error('Invalid column definition: expect a string or an object')]; } } - // Normalize option `quoted_string` - if(options.quoted_string === undefined || options.quoted_string === null){ - options.quoted_string = false; - } - // Normalize option `eof` - if(options.eof === undefined || options.eof === null){ - options.eof = true; - } - // Normalize option `escape` - if(options.escape === undefined || options.escape === null){ - options.escape = '"'; - }else if(isBuffer$1(options.escape)){ - options.escape = options.escape.toString(); - }else if(typeof options.escape !== 'string'){ - return Error(`Invalid Option: escape must be a buffer or a string, got ${JSON.stringify(options.escape)}`); - } - if (options.escape.length > 1){ - return Error(`Invalid Option: escape must be one character, got ${options.escape.length} characters`); - } - // Normalize option `header` - if(options.header === undefined || options.header === null){ - options.header = false; - } - // Normalize option `columns` - const [err, columns] = this.normalize_columns(options.columns); - if(err) return err; - options.columns = columns; - // Normalize option `quoted` - if(options.quoted === undefined || options.quoted === null){ - options.quoted = false; - } - // Normalize option `cast` - if(options.cast === undefined || options.cast === null){ - options.cast = {}; - } - // Normalize option cast.bigint - if(options.cast.bigint === undefined || options.cast.bigint === null){ - // Cast boolean to string by default - options.cast.bigint = value => '' + value; - } - // Normalize option cast.boolean - if(options.cast.boolean === undefined || options.cast.boolean === null){ - // Cast boolean to string by default - options.cast.boolean = value => value ? '1' : ''; - } - // Normalize option cast.date - if(options.cast.date === undefined || options.cast.date === null){ - // Cast date to timestamp string by default - options.cast.date = value => '' + value.getTime(); - } - // Normalize option cast.number - if(options.cast.number === undefined || options.cast.number === null){ - // Cast number to string using native casting by default - options.cast.number = value => '' + value; - } - // Normalize option cast.object - if(options.cast.object === undefined || options.cast.object === null){ - // Stringify object as JSON by default - options.cast.object = value => JSON.stringify(value); - } - // Normalize option cast.string - if(options.cast.string === undefined || options.cast.string === null){ - // Leave string untouched - options.cast.string = function(value){return value;}; - } - // Normalize option `record_delimiter` - if(options.record_delimiter === undefined || options.record_delimiter === null){ - options.record_delimiter = '\n'; - }else if(isBuffer$1(options.record_delimiter)){ - options.record_delimiter = options.record_delimiter.toString(); - }else if(typeof options.record_delimiter !== 'string'){ - return Error(`Invalid Option: record_delimiter must be a buffer or a string, got ${JSON.stringify(options.record_delimiter)}`); - } + columns = newcolumns; } - _transform(chunk, encoding, callback){ - if(this.state.stop === true){ - return; - } - const err = this.__transform(chunk); - if(err !== undefined){ - this.state.stop = true; + return [undefined, columns]; + }; + + const underscore = function(str){ + return str.replace(/([A-Z])/g, function(_, match){ + return '_' + match.toLowerCase(); + }); + }; + + const normalize_options = function(opts) { + const options = {}; + // Merge with user options + for(const opt in opts){ + options[underscore(opt)] = opts[opt]; + } + // Normalize option `bom` + if(options.bom === undefined || options.bom === null || options.bom === false){ + options.bom = false; + }else if(options.bom !== true){ + return [new CsvError('CSV_OPTION_BOOLEAN_INVALID_TYPE', [ + 'option `bom` is optional and must be a boolean value,', + `got ${JSON.stringify(options.bom)}` + ])]; + } + // Normalize option `delimiter` + if(options.delimiter === undefined || options.delimiter === null){ + options.delimiter = ','; + }else if(isBuffer$1(options.delimiter)){ + options.delimiter = options.delimiter.toString(); + }else if(typeof options.delimiter !== 'string'){ + return [new CsvError('CSV_OPTION_DELIMITER_INVALID_TYPE', [ + 'option `delimiter` must be a buffer or a string,', + `got ${JSON.stringify(options.delimiter)}` + ])]; + } + // Normalize option `quote` + if(options.quote === undefined || options.quote === null){ + options.quote = '"'; + }else if(options.quote === true){ + options.quote = '"'; + }else if(options.quote === false){ + options.quote = ''; + }else if (isBuffer$1(options.quote)){ + options.quote = options.quote.toString(); + }else if(typeof options.quote !== 'string'){ + return [new CsvError('CSV_OPTION_QUOTE_INVALID_TYPE', [ + 'option `quote` must be a boolean, a buffer or a string,', + `got ${JSON.stringify(options.quote)}` + ])]; + } + // Normalize option `quoted` + if(options.quoted === undefined || options.quoted === null){ + options.quoted = false; + } + // Normalize option `quoted_empty` + if(options.quoted_empty === undefined || options.quoted_empty === null){ + options.quoted_empty = undefined; + } + // Normalize option `quoted_match` + if(options.quoted_match === undefined || options.quoted_match === null || options.quoted_match === false){ + options.quoted_match = null; + }else if(!Array.isArray(options.quoted_match)){ + options.quoted_match = [options.quoted_match]; + } + if(options.quoted_match){ + for(const quoted_match of options.quoted_match){ + const isString = typeof quoted_match === 'string'; + const isRegExp = quoted_match instanceof RegExp; + if(!isString && !isRegExp){ + return [Error(`Invalid Option: quoted_match must be a string or a regex, got ${JSON.stringify(quoted_match)}`)]; + } } - callback(err); } - _flush(callback){ - if(this.state.stop === true){ - // Note, Node.js 12 call flush even after an error, we must prevent - // `callback` from being called in flush without any error. - return; - } - if(this.info.records === 0){ - this.bom(); - const err = this.headers(); - if(err) callback(err); - } - callback(); + // Normalize option `quoted_string` + if(options.quoted_string === undefined || options.quoted_string === null){ + options.quoted_string = false; + } + // Normalize option `eof` + if(options.eof === undefined || options.eof === null){ + options.eof = true; + } + // Normalize option `escape` + if(options.escape === undefined || options.escape === null){ + options.escape = '"'; + }else if(isBuffer$1(options.escape)){ + options.escape = options.escape.toString(); + }else if(typeof options.escape !== 'string'){ + return [Error(`Invalid Option: escape must be a buffer or a string, got ${JSON.stringify(options.escape)}`)]; + } + if (options.escape.length > 1){ + return [Error(`Invalid Option: escape must be one character, got ${options.escape.length} characters`)]; + } + // Normalize option `header` + if(options.header === undefined || options.header === null){ + options.header = false; + } + // Normalize option `columns` + const [errColumns, columns] = normalize_columns(options.columns); + if(errColumns !== undefined) return [errColumns]; + options.columns = columns; + // Normalize option `quoted` + if(options.quoted === undefined || options.quoted === null){ + options.quoted = false; + } + // Normalize option `cast` + if(options.cast === undefined || options.cast === null){ + options.cast = {}; + } + // Normalize option cast.bigint + if(options.cast.bigint === undefined || options.cast.bigint === null){ + // Cast boolean to string by default + options.cast.bigint = value => '' + value; + } + // Normalize option cast.boolean + if(options.cast.boolean === undefined || options.cast.boolean === null){ + // Cast boolean to string by default + options.cast.boolean = value => value ? '1' : ''; + } + // Normalize option cast.date + if(options.cast.date === undefined || options.cast.date === null){ + // Cast date to timestamp string by default + options.cast.date = value => '' + value.getTime(); + } + // Normalize option cast.number + if(options.cast.number === undefined || options.cast.number === null){ + // Cast number to string using native casting by default + options.cast.number = value => '' + value; + } + // Normalize option cast.object + if(options.cast.object === undefined || options.cast.object === null){ + // Stringify object as JSON by default + options.cast.object = value => JSON.stringify(value); + } + // Normalize option cast.string + if(options.cast.string === undefined || options.cast.string === null){ + // Leave string untouched + options.cast.string = function(value){return value;}; + } + // Normalize option `on_record` + if(options.on_record !== undefined && typeof options.on_record !== 'function'){ + return [Error(`Invalid Option: "on_record" must be a function.`)]; + } + // Normalize option `record_delimiter` + if(options.record_delimiter === undefined || options.record_delimiter === null){ + options.record_delimiter = '\n'; + }else if(isBuffer$1(options.record_delimiter)){ + options.record_delimiter = options.record_delimiter.toString(); + }else if(typeof options.record_delimiter !== 'string'){ + return [Error(`Invalid Option: record_delimiter must be a buffer or a string, got ${JSON.stringify(options.record_delimiter)}`)]; + } + switch(options.record_delimiter){ + case 'auto': + options.record_delimiter = null; + break; + case 'unix': + options.record_delimiter = "\n"; + break; + case 'mac': + options.record_delimiter = "\r"; + break; + case 'windows': + options.record_delimiter = "\r\n"; + break; + case 'ascii': + options.record_delimiter = "\u001e"; + break; + case 'unicode': + options.record_delimiter = "\u2028"; + break; } - __transform(chunk){ - // Chunk validation - if(!Array.isArray(chunk) && typeof chunk !== 'object'){ - return Error(`Invalid Record: expect an array or an object, got ${JSON.stringify(chunk)}`); - } - // Detect columns from the first record - if(this.info.records === 0){ - if(Array.isArray(chunk)){ - if(this.options.header === true && this.options.columns === undefined){ - return Error('Undiscoverable Columns: header option requires column option or object records'); + return [undefined, options]; + }; + + const bom_utf8 = Buffer.from([239, 187, 191]); + + const stringifier = function(options, state, info){ + return { + options: options, + state: state, + info: info, + __transform: function(chunk, push){ + // Chunk validation + if(!Array.isArray(chunk) && typeof chunk !== 'object'){ + return Error(`Invalid Record: expect an array or an object, got ${JSON.stringify(chunk)}`); + } + // Detect columns from the first record + if(this.info.records === 0){ + if(Array.isArray(chunk)){ + if(this.options.header === true && this.options.columns === undefined){ + return Error('Undiscoverable Columns: header option requires column option or object records'); + } + }else if(this.options.columns === undefined){ + const [err, columns] = normalize_columns(Object.keys(chunk)); + if(err) return; + this.options.columns = columns; } - }else if(this.options.columns === undefined){ - const [err, columns] = this.normalize_columns(Object.keys(chunk)); - if(err) return; - this.options.columns = columns; } - } - // Emit the header - if(this.info.records === 0){ - this.bom(); - const err = this.headers(); - if(err) return err; - } - // Emit and stringify the record if an object or an array - try{ - this.emit('record', chunk, this.info.records); - }catch(err){ - return err; - } - // Convert the record into a string - let err, chunk_string; - if(this.options.eof){ - [err, chunk_string] = this.stringify(chunk); - if(err) return err; - if(chunk_string === undefined){ - return; - }else { - chunk_string = chunk_string + this.options.record_delimiter; + // Emit the header + if(this.info.records === 0){ + this.bom(push); + const err = this.headers(push); + if(err) return err; } - }else { - [err, chunk_string] = this.stringify(chunk); - if(err) return err; - if(chunk_string === undefined){ - return; - }else { - if(this.options.header || this.info.records){ - chunk_string = this.options.record_delimiter + chunk_string; + // Emit and stringify the record if an object or an array + try{ + // this.emit('record', chunk, this.info.records); + if(this.options.on_record){ + this.options.on_record(chunk, this.info.records); } + }catch(err){ + return err; } - } - // Emit the csv - this.info.records++; - this.push(chunk_string); - } - stringify(chunk, chunkIsHeader=false){ - if(typeof chunk !== 'object'){ - return [undefined, chunk]; - } - const {columns} = this.options; - const record = []; - // Record is an array - if(Array.isArray(chunk)){ - // We are getting an array but the user has specified output columns. In - // this case, we respect the columns indexes - if(columns){ - chunk.splice(columns.length); - } - // Cast record elements - for(let i=0; i= 0; - const containsQuote = (quote !== '') && value.indexOf(quote) >= 0; - const containsEscape = value.indexOf(escape) >= 0 && (escape !== quote); - const containsRecordDelimiter = value.indexOf(record_delimiter) >= 0; - const quotedString = quoted_string && typeof field === 'string'; - let quotedMatch = quoted_match && quoted_match.filter(quoted_match => { - if(typeof quoted_match === 'string'){ - return value.indexOf(quoted_match) !== -1; - }else { - return quoted_match.test(value); + } + let csvrecord = ''; + for(let i=0; i 0; - const shouldQuote = containsQuote === true || containsdelimiter || containsRecordDelimiter || quoted || quotedString || quotedMatch; - if(shouldQuote === true && containsEscape === true){ - const regexp = escape === '\\' - ? new RegExp(escape + escape, 'g') - : new RegExp(escape, 'g'); - value = value.replace(regexp, escape + escape); + options = {...this.options, ...options}; + [err, options] = normalize_options(options); + if(err !== undefined){ + return [err]; + } + }else if(value === undefined || value === null){ + options = this.options; + }else { + return [Error(`Invalid Casting Value: returned value must return a string, an object, null or undefined, got ${JSON.stringify(value)}`)]; } - if(containsQuote === true){ - const regexp = new RegExp(quote,'g'); - value = value.replace(regexp, escape + quote); + const {delimiter, escape, quote, quoted, quoted_empty, quoted_string, quoted_match, record_delimiter} = options; + if(value){ + if(typeof value !== 'string'){ + return [Error(`Formatter must return a string, null or undefined, got ${JSON.stringify(value)}`)]; + } + const containsdelimiter = delimiter.length && value.indexOf(delimiter) >= 0; + const containsQuote = (quote !== '') && value.indexOf(quote) >= 0; + const containsEscape = value.indexOf(escape) >= 0 && (escape !== quote); + const containsRecordDelimiter = value.indexOf(record_delimiter) >= 0; + const quotedString = quoted_string && typeof field === 'string'; + let quotedMatch = quoted_match && quoted_match.filter(quoted_match => { + if(typeof quoted_match === 'string'){ + return value.indexOf(quoted_match) !== -1; + }else { + return quoted_match.test(value); + } + }); + quotedMatch = quotedMatch && quotedMatch.length > 0; + const shouldQuote = containsQuote === true || containsdelimiter || containsRecordDelimiter || quoted || quotedString || quotedMatch; + if(shouldQuote === true && containsEscape === true){ + const regexp = escape === '\\' + ? new RegExp(escape + escape, 'g') + : new RegExp(escape, 'g'); + value = value.replace(regexp, escape + escape); + } + if(containsQuote === true){ + const regexp = new RegExp(quote,'g'); + value = value.replace(regexp, escape + quote); + } + if(shouldQuote === true){ + value = quote + value + quote; + } + csvrecord += value; + }else if(quoted_empty === true || (field === '' && quoted_string === true && quoted_empty !== false)){ + csvrecord += quote + quote; } - if(shouldQuote === true){ - value = quote + value + quote; + if(i !== record.length - 1){ + csvrecord += delimiter; } - csvrecord += value; - }else if(quoted_empty === true || (field === '' && quoted_string === true && quoted_empty !== false)){ - csvrecord += quote + quote; } - if(i !== record.length - 1){ - csvrecord += delimiter; + return [undefined, csvrecord]; + }, + bom: function(push){ + if(this.options.bom !== true){ + return; + } + push(bom_utf8); + }, + headers: function(push){ + if(this.options.header === false){ + return; + } + if(this.options.columns === undefined){ + return; + } + let err; + let headers = this.options.columns.map(column => column.header); + if(this.options.eof){ + [err, headers] = this.stringify(headers, true); + headers += this.options.record_delimiter; + }else { + [err, headers] = this.stringify(headers); + } + if(err) return err; + push(headers); + }, + __cast: function(value, context){ + const type = typeof value; + try{ + if(type === 'string'){ // Fine for 99% of the cases + return [undefined, this.options.cast.string(value, context)]; + }else if(type === 'bigint'){ + return [undefined, this.options.cast.bigint(value, context)]; + }else if(type === 'number'){ + return [undefined, this.options.cast.number(value, context)]; + }else if(type === 'boolean'){ + return [undefined, this.options.cast.boolean(value, context)]; + }else if(value instanceof Date){ + return [undefined, this.options.cast.date(value, context)]; + }else if(type === 'object' && value !== null){ + return [undefined, this.options.cast.object(value, context)]; + }else { + return [undefined, value, value]; + } + }catch(err){ + return [err]; } } - return [undefined, csvrecord]; - } - bom(){ - if(this.options.bom !== true){ - return; - } - this.push(bom_utf8); + }; + }; + + class Stringifier extends Transform { + constructor(opts = {}){ + super({...{writableObjectMode: true}, ...opts}); + const [err, options] = normalize_options(opts); + if(err !== undefined) throw err; + // Expose options + this.options = options; + // Internal state + this.state = { + stop: false + }; + // Information + this.info = { + records: 0 + }; + this.api = stringifier(this.options, this.state, this.info); + this.api.options.on_record = (...args) => { + this.emit('record', ...args); + }; } - headers(){ - if(this.options.header === false){ - return; - } - if(this.options.columns === undefined){ + _transform(chunk, encoding, callback){ + if(this.state.stop === true){ return; } - let err; - let headers = this.options.columns.map(column => column.header); - if(this.options.eof){ - [err, headers] = this.stringify(headers, true); - headers += this.options.record_delimiter; - }else { - [err, headers] = this.stringify(headers); - } - if(err) return err; - this.push(headers); - } - __cast(value, context){ - const type = typeof value; - try{ - if(type === 'string'){ // Fine for 99% of the cases - return [undefined, this.options.cast.string(value, context)]; - }else if(type === 'bigint'){ - return [undefined, this.options.cast.bigint(value, context)]; - }else if(type === 'number'){ - return [undefined, this.options.cast.number(value, context)]; - }else if(type === 'boolean'){ - return [undefined, this.options.cast.boolean(value, context)]; - }else if(value instanceof Date){ - return [undefined, this.options.cast.date(value, context)]; - }else if(type === 'object' && value !== null){ - return [undefined, this.options.cast.object(value, context)]; - }else { - return [undefined, value, value]; - } - }catch(err){ - return [err]; + const err = this.api.__transform(chunk, this.push.bind(this)); + if(err !== undefined){ + this.state.stop = true; } + callback(err); } - normalize_columns(columns){ - if(columns === undefined || columns === null){ - return []; - } - if(typeof columns !== 'object'){ - return [Error('Invalid option "columns": expect an array or an object')]; + _flush(callback){ + if(this.state.stop === true){ + // Note, Node.js 12 call flush even after an error, we must prevent + // `callback` from being called in flush without any error. + return; } - if(!Array.isArray(columns)){ - const newcolumns = []; - for(const k in columns){ - newcolumns.push({ - key: k, - header: columns[k] - }); - } - columns = newcolumns; - }else { - const newcolumns = []; - for(const column of columns){ - if(typeof column === 'string'){ - newcolumns.push({ - key: column, - header: column - }); - }else if(typeof column === 'object' && column !== undefined && !Array.isArray(column)){ - if(!column.key){ - return [Error('Invalid column definition: property "key" is required')]; - } - if(column.header === undefined){ - column.header = column.key; - } - newcolumns.push(column); - }else { - return [Error('Invalid column definition: expect a string or an object')]; - } - } - columns = newcolumns; + if(this.info.records === 0){ + this.api.bom(this.push.bind(this)); + const err = this.api.headers(this.push.bind(this)); + if(err) callback(err); } - return [undefined, columns]; + callback(); } } @@ -7279,7 +7366,7 @@ const type = typeof argument; if(data === undefined && (Array.isArray(argument))){ data = argument; - }else if(options === undefined && isObject(argument)){ + }else if(options === undefined && is_object(argument)){ options = argument; }else if(callback === undefined && type === 'function'){ callback = argument; diff --git a/packages/csv/dist/umd/sync.js b/packages/csv/dist/umd/sync.js index c77783a85..1734a972c 100644 --- a/packages/csv/dist/umd/sync.js +++ b/packages/csv/dist/umd/sync.js @@ -118,7 +118,7 @@ return parts.join('') } - function read (buffer, offset, isLE, mLen, nBytes) { + function read$1 (buffer, offset, isLE, mLen, nBytes) { var e, m; var eLen = nBytes * 8 - mLen - 1; var eMax = (1 << eLen) - 1; @@ -1399,22 +1399,22 @@ Buffer.prototype.readFloatLE = function readFloatLE (offset, noAssert) { if (!noAssert) checkOffset(offset, 4, this.length); - return read(this, offset, true, 23, 4) + return read$1(this, offset, true, 23, 4) }; Buffer.prototype.readFloatBE = function readFloatBE (offset, noAssert) { if (!noAssert) checkOffset(offset, 4, this.length); - return read(this, offset, false, 23, 4) + return read$1(this, offset, false, 23, 4) }; Buffer.prototype.readDoubleLE = function readDoubleLE (offset, noAssert) { if (!noAssert) checkOffset(offset, 8, this.length); - return read(this, offset, true, 52, 8) + return read$1(this, offset, true, 52, 8) }; Buffer.prototype.readDoubleBE = function readDoubleBE (offset, noAssert) { if (!noAssert) checkOffset(offset, 8, this.length); - return read(this, offset, false, 52, 8) + return read$1(this, offset, false, 52, 8) }; function checkInt (buf, value, offset, ext, max, min) { @@ -2633,7 +2633,7 @@ } }); for (var x = args[i]; i < len; x = args[++i]) { - if (isNull(x) || !isObject$2(x)) { + if (isNull(x) || !isObject(x)) { str += ' ' + x; } else { str += ' ' + inspect(x); @@ -3049,19 +3049,19 @@ } function isRegExp(re) { - return isObject$2(re) && objectToString(re) === '[object RegExp]'; + return isObject(re) && objectToString(re) === '[object RegExp]'; } - function isObject$2(arg) { + function isObject(arg) { return typeof arg === 'object' && arg !== null; } function isDate(d) { - return isObject$2(d) && objectToString(d) === '[object Date]'; + return isObject(d) && objectToString(d) === '[object Date]'; } function isError(e) { - return isObject$2(e) && + return isObject(e) && (objectToString(e) === '[object Error]' || e instanceof Error); } @@ -3112,7 +3112,7 @@ function _extend(origin, add) { // Don't do anything if add isn't an object - if (!add || !isObject$2(add)) return origin; + if (!add || !isObject(add)) return origin; var keys = Object.keys(add); var i = keys.length; @@ -3134,7 +3134,7 @@ isFunction: isFunction, isError: isError, isDate: isDate, - isObject: isObject$2, + isObject: isObject, isRegExp: isRegExp, isUndefined: isUndefined, isSymbol: isSymbol$1, @@ -5042,20 +5042,64 @@ return dest; }; - const Generator = function(options = {}){ + const init_state$1 = (options) => { + // State + return { + start_time: options.duration ? Date.now() : null, + fixed_size_buffer: '', + count_written: 0, + count_created: 0, + }; + }; + + // Generate a random number between 0 and 1 with 2 decimals. The function is idempotent if it detect the "seed" option. + const random = function(options={}){ + if(options.seed){ + return options.seed = options.seed * Math.PI * 100 % 100 / 100; + }else { + return Math.random(); + } + }; + + const types = { + // Generate an ASCII value. + ascii: function({options}){ + const column = []; + const nb_chars = Math.ceil(random(options) * options.maxWordLength); + for(let i=0; i { // Convert Stream Readable options if underscored - if(options.high_water_mark){ - options.highWaterMark = options.high_water_mark; + if(opts.high_water_mark){ + opts.highWaterMark = opts.high_water_mark; } - if(options.object_mode){ - options.objectMode = options.object_mode; + if(opts.object_mode){ + opts.objectMode = opts.object_mode; } - // Call parent constructor - Stream.Readable.call(this, options); // Clone and camelize options - this.options = {}; - for(const k in options){ - this.options[Generator.camelize(k)] = options[k]; + const options = {}; + for(const k in opts){ + options[camelize(k)] = opts[k]; } // Normalize options const dft = { @@ -5073,110 +5117,100 @@ sleep: 0, }; for(const k in dft){ - if(this.options[k] === undefined){ - this.options[k] = dft[k]; + if(options[k] === undefined){ + options[k] = dft[k]; } } // Default values - if(this.options.eof === true){ - this.options.eof = this.options.rowDelimiter; + if(options.eof === true){ + options.eof = options.rowDelimiter; } - // State - this._ = { - start_time: this.options.duration ? Date.now() : null, - fixed_size_buffer: '', - count_written: 0, - count_created: 0, - }; - if(typeof this.options.columns === 'number'){ - this.options.columns = new Array(this.options.columns); + if(typeof options.columns === 'number'){ + options.columns = new Array(options.columns); } - const accepted_header_types = Object.keys(Generator).filter((t) => (!['super_', 'camelize'].includes(t))); - for(let i = 0; i < this.options.columns.length; i++){ - const v = this.options.columns[i] || 'ascii'; + const accepted_header_types = Object.keys(types).filter((t) => (!['super_', 'camelize'].includes(t))); + for(let i = 0; i < options.columns.length; i++){ + const v = options.columns[i] || 'ascii'; if(typeof v === 'string'){ if(!accepted_header_types.includes(v)){ throw Error(`Invalid column type: got "${v}", default values are ${JSON.stringify(accepted_header_types)}`); } - this.options.columns[i] = Generator[v]; + options.columns[i] = types[v]; } } - return this; + return options; }; - util.inherits(Generator, Stream.Readable); - // Generate a random number between 0 and 1 with 2 decimals. The function is idempotent if it detect the "seed" option. - Generator.prototype.random = function(){ - if(this.options.seed){ - return this.options.seed = this.options.seed * Math.PI * 100 % 100 / 100; - }else { - return Math.random(); - } - }; - // Stop the generation. - Generator.prototype.end = function(){ - this.push(null); - }; - // Put new data into the read queue. - Generator.prototype._read = function(size){ + const read = (options, state, size, push, close) => { // Already started const data = []; - let length = this._.fixed_size_buffer.length; + let length = state.fixed_size_buffer.length; if(length !== 0){ - data.push(this._.fixed_size_buffer); + data.push(state.fixed_size_buffer); } // eslint-disable-next-line while(true){ // Time for some rest: flush first and stop later - if((this._.count_created === this.options.length) || (this.options.end && Date.now() > this.options.end) || (this.options.duration && Date.now() > this._.start_time + this.options.duration)){ + if((state.count_created === options.length) || (options.end && Date.now() > options.end) || (options.duration && Date.now() > state.start_time + options.duration)){ // Flush if(data.length){ - if(this.options.objectMode){ + if(options.objectMode){ for(const record of data){ - this.__push(record); + push(record); } }else { - this.__push(data.join('') + (this.options.eof ? this.options.eof : '')); + push(data.join('') + (options.eof ? options.eof : '')); } - this._.end = true; + state.end = true; }else { - this.push(null); + close(); } return; } // Create the record let record = []; let recordLength; - this.options.columns.forEach((fn) => { - record.push(fn(this)); + options.columns.forEach((fn) => { + const result = fn({options: options, state: state}); + const type = typeof result; + if(result !== null && type !== 'string' && type !== 'number'){ + throw Error([ + 'INVALID_VALUE:', + 'values returned by column function must be', + 'a string, a number or null,', + `got ${JSON.stringify(result)}` + ].join(' ')); + } + record.push(result); }); // Obtain record length - if(this.options.objectMode){ + if(options.objectMode){ recordLength = 0; // recordLength is currently equal to the number of columns // This is wrong and shall equal to 1 record only - for(const column of record) + for(const column of record){ recordLength += column.length; + } }else { // Stringify the record - record = (this._.count_created === 0 ? '' : this.options.rowDelimiter)+record.join(this.options.delimiter); + record = (state.count_created === 0 ? '' : options.rowDelimiter)+record.join(options.delimiter); recordLength = record.length; } - this._.count_created++; + state.count_created++; if(length + recordLength > size){ - if(this.options.objectMode){ + if(options.objectMode){ data.push(record); for(const record of data){ - this.__push(record); + push(record); } }else { - if(this.options.fixedSize){ - this._.fixed_size_buffer = record.substr(size - length); + if(options.fixedSize){ + state.fixed_size_buffer = record.substr(size - length); data.push(record.substr(0, size - length)); }else { data.push(record); } - this.__push(data.join('')); + push(data.join('')); } return; } @@ -5184,42 +5218,41 @@ data.push(record); } }; + + const Generator = function(options = {}){ + this.options = normalize_options$2(options); + // Call parent constructor + Stream.Readable.call(this, this.options); + this.state = init_state$1(this.options); + return this; + }; + util.inherits(Generator, Stream.Readable); + + // Stop the generation. + Generator.prototype.end = function(){ + this.push(null); + }; + // Put new data into the read queue. + Generator.prototype._read = function(size){ + const self = this; + read(this.options, this.state, size, function(chunk) { + self.__push(chunk); + }, function(){ + self.push(null); + }); + }; // Put new data into the read queue. Generator.prototype.__push = function(record){ + // console.log('push', record) const push = () => { - this._.count_written++; + this.state.count_written++; this.push(record); - if(this._.end === true){ + if(this.state.end === true){ return this.push(null); } }; this.options.sleep > 0 ? setTimeout(push, this.options.sleep) : push(); }; - // Generate an ASCII value. - Generator.ascii = function(gen){ - // Column - const column = []; - const nb_chars = Math.ceil(gen.random() * gen.options.maxWordLength); - for(let i=0; i delimiter.length), + // Skip if the remaining buffer can be escape sequence + options.quote !== null ? options.quote.length : 0, + ), + previousBuf: undefined, + quoting: false, + stop: false, + rawBuffer: new ResizeableBuffer(100), + record: [], + recordHasError: false, + record_length: 0, + recordDelimiterMaxLength: options.record_delimiter.length === 0 ? 2 : Math.max(...options.record_delimiter.map((v) => v.length)), + trimChars: [Buffer.from(' ', options.encoding)[0], Buffer.from('\t', options.encoding)[0]], + wasQuoting: false, + wasRowDelimiter: false + }; }; - class CsvError$1 extends Error { - constructor(code, message, options, ...contexts) { - if(Array.isArray(message)) message = message.join(' '); - super(message); - if(Error.captureStackTrace !== undefined){ - Error.captureStackTrace(this, CsvError$1); - } - this.code = code; - for(const context of contexts){ - for(const key in context){ - const value = context[key]; - this[key] = isBuffer$1(value) ? value.toString(options.encoding) : value == null ? value : JSON.parse(JSON.stringify(value)); - } - } - } - } - const underscore$1 = function(str){ return str.replace(/([A-Z])/g, function(_, match){ return '_' + match.toLowerCase(); }); }; - const isObject$1 = function(obj){ - return (typeof obj === 'object' && obj !== null && !Array.isArray(obj)); - }; - - const isRecordEmpty = function(record){ - return record.every((field) => field == null || field.toString && field.toString().trim() === ''); - }; - - const normalizeColumnsArray = function(columns){ - const normalizedColumns = []; - for(let i = 0, l = columns.length; i < l; i++){ - const column = columns[i]; - if(column === undefined || column === null || column === false){ - normalizedColumns[i] = { disabled: true }; - }else if(typeof column === 'string'){ - normalizedColumns[i] = { name: column }; - }else if(isObject$1(column)){ - if(typeof column.name !== 'string'){ - throw new CsvError$1('CSV_OPTION_COLUMNS_MISSING_NAME', [ - 'Option columns missing name:', - `property "name" is required at position ${i}`, - 'when column is an object literal' - ]); - } - normalizedColumns[i] = column; - }else { - throw new CsvError$1('CSV_INVALID_COLUMN_DEFINITION', [ - 'Invalid column definition:', - 'expect a string or a literal object,', - `got ${JSON.stringify(column)} at position ${i}` - ]); - } - } - return normalizedColumns; - }; - - class Parser extends Transform { - constructor(opts = {}){ - super({...{readableObjectMode: true}, ...opts, encoding: null}); - this.__originalOptions = opts; - this.__normalizeOptions(opts); - } - __normalizeOptions(opts){ - const options = {}; - // Merge with user options - for(const opt in opts){ - options[underscore$1(opt)] = opts[opt]; - } - // Normalize option `encoding` - // Note: defined first because other options depends on it - // to convert chars/strings into buffers. - if(options.encoding === undefined || options.encoding === true){ - options.encoding = 'utf8'; - }else if(options.encoding === null || options.encoding === false){ - options.encoding = null; - }else if(typeof options.encoding !== 'string' && options.encoding !== null){ - throw new CsvError$1('CSV_INVALID_OPTION_ENCODING', [ - 'Invalid option encoding:', - 'encoding must be a string or null to return a buffer,', - `got ${JSON.stringify(options.encoding)}` - ], options); - } - // Normalize option `bom` - if(options.bom === undefined || options.bom === null || options.bom === false){ - options.bom = false; - }else if(options.bom !== true){ - throw new CsvError$1('CSV_INVALID_OPTION_BOM', [ - 'Invalid option bom:', 'bom must be true,', - `got ${JSON.stringify(options.bom)}` - ], options); - } - // Normalize option `cast` - let fnCastField = null; - if(options.cast === undefined || options.cast === null || options.cast === false || options.cast === ''){ - options.cast = undefined; - }else if(typeof options.cast === 'function'){ - fnCastField = options.cast; - options.cast = true; - }else if(options.cast !== true){ - throw new CsvError$1('CSV_INVALID_OPTION_CAST', [ - 'Invalid option cast:', 'cast must be true or a function,', - `got ${JSON.stringify(options.cast)}` - ], options); - } - // Normalize option `cast_date` - if(options.cast_date === undefined || options.cast_date === null || options.cast_date === false || options.cast_date === ''){ - options.cast_date = false; - }else if(options.cast_date === true){ - options.cast_date = function(value){ - const date = Date.parse(value); - return !isNaN(date) ? new Date(date) : value; - }; - }else { - throw new CsvError$1('CSV_INVALID_OPTION_CAST_DATE', [ - 'Invalid option cast_date:', 'cast_date must be true or a function,', - `got ${JSON.stringify(options.cast_date)}` - ], options); - } - // Normalize option `columns` - let fnFirstLineToHeaders = null; - if(options.columns === true){ - // Fields in the first line are converted as-is to columns - fnFirstLineToHeaders = undefined; - }else if(typeof options.columns === 'function'){ - fnFirstLineToHeaders = options.columns; - options.columns = true; - }else if(Array.isArray(options.columns)){ - options.columns = normalizeColumnsArray(options.columns); - }else if(options.columns === undefined || options.columns === null || options.columns === false){ - options.columns = false; - }else { - throw new CsvError$1('CSV_INVALID_OPTION_COLUMNS', [ - 'Invalid option columns:', - 'expect an array, a function or true,', - `got ${JSON.stringify(options.columns)}` + const normalize_options$1 = function(opts){ + const options = {}; + // Merge with user options + for(const opt in opts){ + options[underscore$1(opt)] = opts[opt]; + } + // Normalize option `encoding` + // Note: defined first because other options depends on it + // to convert chars/strings into buffers. + if(options.encoding === undefined || options.encoding === true){ + options.encoding = 'utf8'; + }else if(options.encoding === null || options.encoding === false){ + options.encoding = null; + }else if(typeof options.encoding !== 'string' && options.encoding !== null){ + throw new CsvError$1('CSV_INVALID_OPTION_ENCODING', [ + 'Invalid option encoding:', + 'encoding must be a string or null to return a buffer,', + `got ${JSON.stringify(options.encoding)}` + ], options); + } + // Normalize option `bom` + if(options.bom === undefined || options.bom === null || options.bom === false){ + options.bom = false; + }else if(options.bom !== true){ + throw new CsvError$1('CSV_INVALID_OPTION_BOM', [ + 'Invalid option bom:', 'bom must be true,', + `got ${JSON.stringify(options.bom)}` + ], options); + } + // Normalize option `cast` + options.cast_function = null; + if(options.cast === undefined || options.cast === null || options.cast === false || options.cast === ''){ + options.cast = undefined; + }else if(typeof options.cast === 'function'){ + options.cast_function = options.cast; + options.cast = true; + }else if(options.cast !== true){ + throw new CsvError$1('CSV_INVALID_OPTION_CAST', [ + 'Invalid option cast:', 'cast must be true or a function,', + `got ${JSON.stringify(options.cast)}` + ], options); + } + // Normalize option `cast_date` + if(options.cast_date === undefined || options.cast_date === null || options.cast_date === false || options.cast_date === ''){ + options.cast_date = false; + }else if(options.cast_date === true){ + options.cast_date = function(value){ + const date = Date.parse(value); + return !isNaN(date) ? new Date(date) : value; + }; + }else { + throw new CsvError$1('CSV_INVALID_OPTION_CAST_DATE', [ + 'Invalid option cast_date:', 'cast_date must be true or a function,', + `got ${JSON.stringify(options.cast_date)}` + ], options); + } + // Normalize option `columns` + options.cast_first_line_to_header = null; + if(options.columns === true){ + // Fields in the first line are converted as-is to columns + options.cast_first_line_to_header = undefined; + }else if(typeof options.columns === 'function'){ + options.cast_first_line_to_header = options.columns; + options.columns = true; + }else if(Array.isArray(options.columns)){ + options.columns = normalize_columns_array(options.columns); + }else if(options.columns === undefined || options.columns === null || options.columns === false){ + options.columns = false; + }else { + throw new CsvError$1('CSV_INVALID_OPTION_COLUMNS', [ + 'Invalid option columns:', + 'expect an array, a function or true,', + `got ${JSON.stringify(options.columns)}` + ], options); + } + // Normalize option `group_columns_by_name` + if(options.group_columns_by_name === undefined || options.group_columns_by_name === null || options.group_columns_by_name === false){ + options.group_columns_by_name = false; + }else if(options.group_columns_by_name !== true){ + throw new CsvError$1('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ + 'Invalid option group_columns_by_name:', + 'expect an boolean,', + `got ${JSON.stringify(options.group_columns_by_name)}` + ], options); + }else if(options.columns === false){ + throw new CsvError$1('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ + 'Invalid option group_columns_by_name:', + 'the `columns` mode must be activated.' + ], options); + } + // Normalize option `comment` + if(options.comment === undefined || options.comment === null || options.comment === false || options.comment === ''){ + options.comment = null; + }else { + if(typeof options.comment === 'string'){ + options.comment = Buffer.from(options.comment, options.encoding); + } + if(!isBuffer$1(options.comment)){ + throw new CsvError$1('CSV_INVALID_OPTION_COMMENT', [ + 'Invalid option comment:', + 'comment must be a buffer or a string,', + `got ${JSON.stringify(options.comment)}` ], options); } - // Normalize option `group_columns_by_name` - if(options.group_columns_by_name === undefined || options.group_columns_by_name === null || options.group_columns_by_name === false){ - options.group_columns_by_name = false; - }else if(options.group_columns_by_name !== true){ - throw new CsvError$1('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ - 'Invalid option group_columns_by_name:', - 'expect an boolean,', - `got ${JSON.stringify(options.group_columns_by_name)}` - ], options); - }else if(options.columns === false){ - throw new CsvError$1('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ - 'Invalid option group_columns_by_name:', - 'the `columns` mode must be activated.' - ], options); + } + // Normalize option `delimiter` + const delimiter_json = JSON.stringify(options.delimiter); + if(!Array.isArray(options.delimiter)) options.delimiter = [options.delimiter]; + if(options.delimiter.length === 0){ + throw new CsvError$1('CSV_INVALID_OPTION_DELIMITER', [ + 'Invalid option delimiter:', + 'delimiter must be a non empty string or buffer or array of string|buffer,', + `got ${delimiter_json}` + ], options); + } + options.delimiter = options.delimiter.map(function(delimiter){ + if(delimiter === undefined || delimiter === null || delimiter === false){ + return Buffer.from(',', options.encoding); } - // Normalize option `comment` - if(options.comment === undefined || options.comment === null || options.comment === false || options.comment === ''){ - options.comment = null; - }else { - if(typeof options.comment === 'string'){ - options.comment = Buffer.from(options.comment, options.encoding); - } - if(!isBuffer$1(options.comment)){ - throw new CsvError$1('CSV_INVALID_OPTION_COMMENT', [ - 'Invalid option comment:', - 'comment must be a buffer or a string,', - `got ${JSON.stringify(options.comment)}` - ], options); - } + if(typeof delimiter === 'string'){ + delimiter = Buffer.from(delimiter, options.encoding); } - // Normalize option `delimiter` - const delimiter_json = JSON.stringify(options.delimiter); - if(!Array.isArray(options.delimiter)) options.delimiter = [options.delimiter]; - if(options.delimiter.length === 0){ + if(!isBuffer$1(delimiter) || delimiter.length === 0){ throw new CsvError$1('CSV_INVALID_OPTION_DELIMITER', [ 'Invalid option delimiter:', 'delimiter must be a non empty string or buffer or array of string|buffer,', `got ${delimiter_json}` ], options); } - options.delimiter = options.delimiter.map(function(delimiter){ - if(delimiter === undefined || delimiter === null || delimiter === false){ - return Buffer.from(',', options.encoding); - } - if(typeof delimiter === 'string'){ - delimiter = Buffer.from(delimiter, options.encoding); - } - if(!isBuffer$1(delimiter) || delimiter.length === 0){ - throw new CsvError$1('CSV_INVALID_OPTION_DELIMITER', [ - 'Invalid option delimiter:', - 'delimiter must be a non empty string or buffer or array of string|buffer,', - `got ${delimiter_json}` - ], options); - } - return delimiter; - }); - // Normalize option `escape` - if(options.escape === undefined || options.escape === true){ - options.escape = Buffer.from('"', options.encoding); - }else if(typeof options.escape === 'string'){ - options.escape = Buffer.from(options.escape, options.encoding); - }else if (options.escape === null || options.escape === false){ - options.escape = null; - } - if(options.escape !== null){ - if(!isBuffer$1(options.escape)){ - throw new Error(`Invalid Option: escape must be a buffer, a string or a boolean, got ${JSON.stringify(options.escape)}`); - } + return delimiter; + }); + // Normalize option `escape` + if(options.escape === undefined || options.escape === true){ + options.escape = Buffer.from('"', options.encoding); + }else if(typeof options.escape === 'string'){ + options.escape = Buffer.from(options.escape, options.encoding); + }else if (options.escape === null || options.escape === false){ + options.escape = null; + } + if(options.escape !== null){ + if(!isBuffer$1(options.escape)){ + throw new Error(`Invalid Option: escape must be a buffer, a string or a boolean, got ${JSON.stringify(options.escape)}`); + } + } + // Normalize option `from` + if(options.from === undefined || options.from === null){ + options.from = 1; + }else { + if(typeof options.from === 'string' && /\d+/.test(options.from)){ + options.from = parseInt(options.from); } - // Normalize option `from` - if(options.from === undefined || options.from === null){ - options.from = 1; - }else { - if(typeof options.from === 'string' && /\d+/.test(options.from)){ - options.from = parseInt(options.from); - } - if(Number.isInteger(options.from)){ - if(options.from < 0){ - throw new Error(`Invalid Option: from must be a positive integer, got ${JSON.stringify(opts.from)}`); - } - }else { - throw new Error(`Invalid Option: from must be an integer, got ${JSON.stringify(options.from)}`); + if(Number.isInteger(options.from)){ + if(options.from < 0){ + throw new Error(`Invalid Option: from must be a positive integer, got ${JSON.stringify(opts.from)}`); } - } - // Normalize option `from_line` - if(options.from_line === undefined || options.from_line === null){ - options.from_line = 1; }else { - if(typeof options.from_line === 'string' && /\d+/.test(options.from_line)){ - options.from_line = parseInt(options.from_line); - } - if(Number.isInteger(options.from_line)){ - if(options.from_line <= 0){ - throw new Error(`Invalid Option: from_line must be a positive integer greater than 0, got ${JSON.stringify(opts.from_line)}`); - } - }else { - throw new Error(`Invalid Option: from_line must be an integer, got ${JSON.stringify(opts.from_line)}`); - } - } - // Normalize options `ignore_last_delimiters` - if(options.ignore_last_delimiters === undefined || options.ignore_last_delimiters === null){ - options.ignore_last_delimiters = false; - }else if(typeof options.ignore_last_delimiters === 'number'){ - options.ignore_last_delimiters = Math.floor(options.ignore_last_delimiters); - if(options.ignore_last_delimiters === 0){ - options.ignore_last_delimiters = false; - } - }else if(typeof options.ignore_last_delimiters !== 'boolean'){ - throw new CsvError$1('CSV_INVALID_OPTION_IGNORE_LAST_DELIMITERS', [ - 'Invalid option `ignore_last_delimiters`:', - 'the value must be a boolean value or an integer,', - `got ${JSON.stringify(options.ignore_last_delimiters)}` - ], options); + throw new Error(`Invalid Option: from must be an integer, got ${JSON.stringify(options.from)}`); } - if(options.ignore_last_delimiters === true && options.columns === false){ - throw new CsvError$1('CSV_IGNORE_LAST_DELIMITERS_REQUIRES_COLUMNS', [ - 'The option `ignore_last_delimiters`', - 'requires the activation of the `columns` option' - ], options); + } + // Normalize option `from_line` + if(options.from_line === undefined || options.from_line === null){ + options.from_line = 1; + }else { + if(typeof options.from_line === 'string' && /\d+/.test(options.from_line)){ + options.from_line = parseInt(options.from_line); } - // Normalize option `info` - if(options.info === undefined || options.info === null || options.info === false){ - options.info = false; - }else if(options.info !== true){ - throw new Error(`Invalid Option: info must be true, got ${JSON.stringify(options.info)}`); - } - // Normalize option `max_record_size` - if(options.max_record_size === undefined || options.max_record_size === null || options.max_record_size === false){ - options.max_record_size = 0; - }else if(Number.isInteger(options.max_record_size) && options.max_record_size >= 0);else if(typeof options.max_record_size === 'string' && /\d+/.test(options.max_record_size)){ - options.max_record_size = parseInt(options.max_record_size); - }else { - throw new Error(`Invalid Option: max_record_size must be a positive integer, got ${JSON.stringify(options.max_record_size)}`); - } - // Normalize option `objname` - if(options.objname === undefined || options.objname === null || options.objname === false){ - options.objname = undefined; - }else if(isBuffer$1(options.objname)){ - if(options.objname.length === 0){ - throw new Error(`Invalid Option: objname must be a non empty buffer`); - } - if(options.encoding === null);else { - options.objname = options.objname.toString(options.encoding); - } - }else if(typeof options.objname === 'string'){ - if(options.objname.length === 0){ - throw new Error(`Invalid Option: objname must be a non empty string`); - } - // Great, nothing to do - }else if(typeof options.objname === 'number');else { - throw new Error(`Invalid Option: objname must be a string or a buffer, got ${options.objname}`); - } - if(options.objname !== undefined){ - if(typeof options.objname === 'number'){ - if(options.columns !== false){ - throw Error('Invalid Option: objname index cannot be combined with columns or be defined as a field'); - } - }else { // A string or a buffer - if(options.columns === false){ - throw Error('Invalid Option: objname field must be combined with columns or be defined as an index'); - } + if(Number.isInteger(options.from_line)){ + if(options.from_line <= 0){ + throw new Error(`Invalid Option: from_line must be a positive integer greater than 0, got ${JSON.stringify(opts.from_line)}`); } + }else { + throw new Error(`Invalid Option: from_line must be an integer, got ${JSON.stringify(opts.from_line)}`); } - // Normalize option `on_record` - if(options.on_record === undefined || options.on_record === null){ - options.on_record = undefined; - }else if(typeof options.on_record !== 'function'){ - throw new CsvError$1('CSV_INVALID_OPTION_ON_RECORD', [ - 'Invalid option `on_record`:', - 'expect a function,', - `got ${JSON.stringify(options.on_record)}` - ], options); + } + // Normalize options `ignore_last_delimiters` + if(options.ignore_last_delimiters === undefined || options.ignore_last_delimiters === null){ + options.ignore_last_delimiters = false; + }else if(typeof options.ignore_last_delimiters === 'number'){ + options.ignore_last_delimiters = Math.floor(options.ignore_last_delimiters); + if(options.ignore_last_delimiters === 0){ + options.ignore_last_delimiters = false; } - // Normalize option `quote` - if(options.quote === null || options.quote === false || options.quote === ''){ - options.quote = null; - }else { - if(options.quote === undefined || options.quote === true){ - options.quote = Buffer.from('"', options.encoding); - }else if(typeof options.quote === 'string'){ - options.quote = Buffer.from(options.quote, options.encoding); + }else if(typeof options.ignore_last_delimiters !== 'boolean'){ + throw new CsvError$1('CSV_INVALID_OPTION_IGNORE_LAST_DELIMITERS', [ + 'Invalid option `ignore_last_delimiters`:', + 'the value must be a boolean value or an integer,', + `got ${JSON.stringify(options.ignore_last_delimiters)}` + ], options); + } + if(options.ignore_last_delimiters === true && options.columns === false){ + throw new CsvError$1('CSV_IGNORE_LAST_DELIMITERS_REQUIRES_COLUMNS', [ + 'The option `ignore_last_delimiters`', + 'requires the activation of the `columns` option' + ], options); + } + // Normalize option `info` + if(options.info === undefined || options.info === null || options.info === false){ + options.info = false; + }else if(options.info !== true){ + throw new Error(`Invalid Option: info must be true, got ${JSON.stringify(options.info)}`); + } + // Normalize option `max_record_size` + if(options.max_record_size === undefined || options.max_record_size === null || options.max_record_size === false){ + options.max_record_size = 0; + }else if(Number.isInteger(options.max_record_size) && options.max_record_size >= 0);else if(typeof options.max_record_size === 'string' && /\d+/.test(options.max_record_size)){ + options.max_record_size = parseInt(options.max_record_size); + }else { + throw new Error(`Invalid Option: max_record_size must be a positive integer, got ${JSON.stringify(options.max_record_size)}`); + } + // Normalize option `objname` + if(options.objname === undefined || options.objname === null || options.objname === false){ + options.objname = undefined; + }else if(isBuffer$1(options.objname)){ + if(options.objname.length === 0){ + throw new Error(`Invalid Option: objname must be a non empty buffer`); + } + if(options.encoding === null);else { + options.objname = options.objname.toString(options.encoding); + } + }else if(typeof options.objname === 'string'){ + if(options.objname.length === 0){ + throw new Error(`Invalid Option: objname must be a non empty string`); + } + // Great, nothing to do + }else if(typeof options.objname === 'number');else { + throw new Error(`Invalid Option: objname must be a string or a buffer, got ${options.objname}`); + } + if(options.objname !== undefined){ + if(typeof options.objname === 'number'){ + if(options.columns !== false){ + throw Error('Invalid Option: objname index cannot be combined with columns or be defined as a field'); } - if(!isBuffer$1(options.quote)){ - throw new Error(`Invalid Option: quote must be a buffer or a string, got ${JSON.stringify(options.quote)}`); + }else { // A string or a buffer + if(options.columns === false){ + throw Error('Invalid Option: objname field must be combined with columns or be defined as an index'); } } - // Normalize option `raw` - if(options.raw === undefined || options.raw === null || options.raw === false){ - options.raw = false; - }else if(options.raw !== true){ - throw new Error(`Invalid Option: raw must be true, got ${JSON.stringify(options.raw)}`); - } - // Normalize option `record_delimiter` - if(options.record_delimiter === undefined){ - options.record_delimiter = []; - }else if(typeof options.record_delimiter === 'string' || isBuffer$1(options.record_delimiter)){ - if(options.record_delimiter.length === 0){ - throw new CsvError$1('CSV_INVALID_OPTION_RECORD_DELIMITER', [ - 'Invalid option `record_delimiter`:', - 'value must be a non empty string or buffer,', - `got ${JSON.stringify(options.record_delimiter)}` - ], options); - } - options.record_delimiter = [options.record_delimiter]; - }else if(!Array.isArray(options.record_delimiter)){ + } + // Normalize option `on_record` + if(options.on_record === undefined || options.on_record === null){ + options.on_record = undefined; + }else if(typeof options.on_record !== 'function'){ + throw new CsvError$1('CSV_INVALID_OPTION_ON_RECORD', [ + 'Invalid option `on_record`:', + 'expect a function,', + `got ${JSON.stringify(options.on_record)}` + ], options); + } + // Normalize option `quote` + if(options.quote === null || options.quote === false || options.quote === ''){ + options.quote = null; + }else { + if(options.quote === undefined || options.quote === true){ + options.quote = Buffer.from('"', options.encoding); + }else if(typeof options.quote === 'string'){ + options.quote = Buffer.from(options.quote, options.encoding); + } + if(!isBuffer$1(options.quote)){ + throw new Error(`Invalid Option: quote must be a buffer or a string, got ${JSON.stringify(options.quote)}`); + } + } + // Normalize option `raw` + if(options.raw === undefined || options.raw === null || options.raw === false){ + options.raw = false; + }else if(options.raw !== true){ + throw new Error(`Invalid Option: raw must be true, got ${JSON.stringify(options.raw)}`); + } + // Normalize option `record_delimiter` + if(options.record_delimiter === undefined){ + options.record_delimiter = []; + }else if(typeof options.record_delimiter === 'string' || isBuffer$1(options.record_delimiter)){ + if(options.record_delimiter.length === 0){ throw new CsvError$1('CSV_INVALID_OPTION_RECORD_DELIMITER', [ 'Invalid option `record_delimiter`:', - 'value must be a string, a buffer or array of string|buffer,', + 'value must be a non empty string or buffer,', `got ${JSON.stringify(options.record_delimiter)}` ], options); } - options.record_delimiter = options.record_delimiter.map(function(rd, i){ - if(typeof rd !== 'string' && ! isBuffer$1(rd)){ - throw new CsvError$1('CSV_INVALID_OPTION_RECORD_DELIMITER', [ - 'Invalid option `record_delimiter`:', - 'value must be a string, a buffer or array of string|buffer', - `at index ${i},`, - `got ${JSON.stringify(rd)}` - ], options); - }else if(rd.length === 0){ - throw new CsvError$1('CSV_INVALID_OPTION_RECORD_DELIMITER', [ - 'Invalid option `record_delimiter`:', - 'value must be a non empty string or buffer', - `at index ${i},`, - `got ${JSON.stringify(rd)}` - ], options); - } - if(typeof rd === 'string'){ - rd = Buffer.from(rd, options.encoding); - } - return rd; - }); - // Normalize option `relax_column_count` - if(typeof options.relax_column_count === 'boolean');else if(options.relax_column_count === undefined || options.relax_column_count === null){ - options.relax_column_count = false; - }else { - throw new Error(`Invalid Option: relax_column_count must be a boolean, got ${JSON.stringify(options.relax_column_count)}`); - } - if(typeof options.relax_column_count_less === 'boolean');else if(options.relax_column_count_less === undefined || options.relax_column_count_less === null){ - options.relax_column_count_less = false; - }else { - throw new Error(`Invalid Option: relax_column_count_less must be a boolean, got ${JSON.stringify(options.relax_column_count_less)}`); - } - if(typeof options.relax_column_count_more === 'boolean');else if(options.relax_column_count_more === undefined || options.relax_column_count_more === null){ - options.relax_column_count_more = false; - }else { - throw new Error(`Invalid Option: relax_column_count_more must be a boolean, got ${JSON.stringify(options.relax_column_count_more)}`); - } - // Normalize option `relax_quotes` - if(typeof options.relax_quotes === 'boolean');else if(options.relax_quotes === undefined || options.relax_quotes === null){ - options.relax_quotes = false; - }else { - throw new Error(`Invalid Option: relax_quotes must be a boolean, got ${JSON.stringify(options.relax_quotes)}`); + options.record_delimiter = [options.record_delimiter]; + }else if(!Array.isArray(options.record_delimiter)){ + throw new CsvError$1('CSV_INVALID_OPTION_RECORD_DELIMITER', [ + 'Invalid option `record_delimiter`:', + 'value must be a string, a buffer or array of string|buffer,', + `got ${JSON.stringify(options.record_delimiter)}` + ], options); + } + options.record_delimiter = options.record_delimiter.map(function(rd, i){ + if(typeof rd !== 'string' && ! isBuffer$1(rd)){ + throw new CsvError$1('CSV_INVALID_OPTION_RECORD_DELIMITER', [ + 'Invalid option `record_delimiter`:', + 'value must be a string, a buffer or array of string|buffer', + `at index ${i},`, + `got ${JSON.stringify(rd)}` + ], options); + }else if(rd.length === 0){ + throw new CsvError$1('CSV_INVALID_OPTION_RECORD_DELIMITER', [ + 'Invalid option `record_delimiter`:', + 'value must be a non empty string or buffer', + `at index ${i},`, + `got ${JSON.stringify(rd)}` + ], options); } - // Normalize option `skip_empty_lines` - if(typeof options.skip_empty_lines === 'boolean');else if(options.skip_empty_lines === undefined || options.skip_empty_lines === null){ - options.skip_empty_lines = false; - }else { - throw new Error(`Invalid Option: skip_empty_lines must be a boolean, got ${JSON.stringify(options.skip_empty_lines)}`); + if(typeof rd === 'string'){ + rd = Buffer.from(rd, options.encoding); } - // Normalize option `skip_records_with_empty_values` - if(typeof options.skip_records_with_empty_values === 'boolean');else if(options.skip_records_with_empty_values === undefined || options.skip_records_with_empty_values === null){ - options.skip_records_with_empty_values = false; - }else { - throw new Error(`Invalid Option: skip_records_with_empty_values must be a boolean, got ${JSON.stringify(options.skip_records_with_empty_values)}`); + return rd; + }); + // Normalize option `relax_column_count` + if(typeof options.relax_column_count === 'boolean');else if(options.relax_column_count === undefined || options.relax_column_count === null){ + options.relax_column_count = false; + }else { + throw new Error(`Invalid Option: relax_column_count must be a boolean, got ${JSON.stringify(options.relax_column_count)}`); + } + if(typeof options.relax_column_count_less === 'boolean');else if(options.relax_column_count_less === undefined || options.relax_column_count_less === null){ + options.relax_column_count_less = false; + }else { + throw new Error(`Invalid Option: relax_column_count_less must be a boolean, got ${JSON.stringify(options.relax_column_count_less)}`); + } + if(typeof options.relax_column_count_more === 'boolean');else if(options.relax_column_count_more === undefined || options.relax_column_count_more === null){ + options.relax_column_count_more = false; + }else { + throw new Error(`Invalid Option: relax_column_count_more must be a boolean, got ${JSON.stringify(options.relax_column_count_more)}`); + } + // Normalize option `relax_quotes` + if(typeof options.relax_quotes === 'boolean');else if(options.relax_quotes === undefined || options.relax_quotes === null){ + options.relax_quotes = false; + }else { + throw new Error(`Invalid Option: relax_quotes must be a boolean, got ${JSON.stringify(options.relax_quotes)}`); + } + // Normalize option `skip_empty_lines` + if(typeof options.skip_empty_lines === 'boolean');else if(options.skip_empty_lines === undefined || options.skip_empty_lines === null){ + options.skip_empty_lines = false; + }else { + throw new Error(`Invalid Option: skip_empty_lines must be a boolean, got ${JSON.stringify(options.skip_empty_lines)}`); + } + // Normalize option `skip_records_with_empty_values` + if(typeof options.skip_records_with_empty_values === 'boolean');else if(options.skip_records_with_empty_values === undefined || options.skip_records_with_empty_values === null){ + options.skip_records_with_empty_values = false; + }else { + throw new Error(`Invalid Option: skip_records_with_empty_values must be a boolean, got ${JSON.stringify(options.skip_records_with_empty_values)}`); + } + // Normalize option `skip_records_with_error` + if(typeof options.skip_records_with_error === 'boolean');else if(options.skip_records_with_error === undefined || options.skip_records_with_error === null){ + options.skip_records_with_error = false; + }else { + throw new Error(`Invalid Option: skip_records_with_error must be a boolean, got ${JSON.stringify(options.skip_records_with_error)}`); + } + // Normalize option `rtrim` + if(options.rtrim === undefined || options.rtrim === null || options.rtrim === false){ + options.rtrim = false; + }else if(options.rtrim !== true){ + throw new Error(`Invalid Option: rtrim must be a boolean, got ${JSON.stringify(options.rtrim)}`); + } + // Normalize option `ltrim` + if(options.ltrim === undefined || options.ltrim === null || options.ltrim === false){ + options.ltrim = false; + }else if(options.ltrim !== true){ + throw new Error(`Invalid Option: ltrim must be a boolean, got ${JSON.stringify(options.ltrim)}`); + } + // Normalize option `trim` + if(options.trim === undefined || options.trim === null || options.trim === false){ + options.trim = false; + }else if(options.trim !== true){ + throw new Error(`Invalid Option: trim must be a boolean, got ${JSON.stringify(options.trim)}`); + } + // Normalize options `trim`, `ltrim` and `rtrim` + if(options.trim === true && opts.ltrim !== false){ + options.ltrim = true; + }else if(options.ltrim !== true){ + options.ltrim = false; + } + if(options.trim === true && opts.rtrim !== false){ + options.rtrim = true; + }else if(options.rtrim !== true){ + options.rtrim = false; + } + // Normalize option `to` + if(options.to === undefined || options.to === null){ + options.to = -1; + }else { + if(typeof options.to === 'string' && /\d+/.test(options.to)){ + options.to = parseInt(options.to); } - // Normalize option `skip_records_with_error` - if(typeof options.skip_records_with_error === 'boolean');else if(options.skip_records_with_error === undefined || options.skip_records_with_error === null){ - options.skip_records_with_error = false; - }else { - throw new Error(`Invalid Option: skip_records_with_error must be a boolean, got ${JSON.stringify(options.skip_records_with_error)}`); - } - // Normalize option `rtrim` - if(options.rtrim === undefined || options.rtrim === null || options.rtrim === false){ - options.rtrim = false; - }else if(options.rtrim !== true){ - throw new Error(`Invalid Option: rtrim must be a boolean, got ${JSON.stringify(options.rtrim)}`); - } - // Normalize option `ltrim` - if(options.ltrim === undefined || options.ltrim === null || options.ltrim === false){ - options.ltrim = false; - }else if(options.ltrim !== true){ - throw new Error(`Invalid Option: ltrim must be a boolean, got ${JSON.stringify(options.ltrim)}`); - } - // Normalize option `trim` - if(options.trim === undefined || options.trim === null || options.trim === false){ - options.trim = false; - }else if(options.trim !== true){ - throw new Error(`Invalid Option: trim must be a boolean, got ${JSON.stringify(options.trim)}`); - } - // Normalize options `trim`, `ltrim` and `rtrim` - if(options.trim === true && opts.ltrim !== false){ - options.ltrim = true; - }else if(options.ltrim !== true){ - options.ltrim = false; - } - if(options.trim === true && opts.rtrim !== false){ - options.rtrim = true; - }else if(options.rtrim !== true){ - options.rtrim = false; - } - // Normalize option `to` - if(options.to === undefined || options.to === null){ - options.to = -1; - }else { - if(typeof options.to === 'string' && /\d+/.test(options.to)){ - options.to = parseInt(options.to); - } - if(Number.isInteger(options.to)){ - if(options.to <= 0){ - throw new Error(`Invalid Option: to must be a positive integer greater than 0, got ${JSON.stringify(opts.to)}`); - } - }else { - throw new Error(`Invalid Option: to must be an integer, got ${JSON.stringify(opts.to)}`); + if(Number.isInteger(options.to)){ + if(options.to <= 0){ + throw new Error(`Invalid Option: to must be a positive integer greater than 0, got ${JSON.stringify(opts.to)}`); } - } - // Normalize option `to_line` - if(options.to_line === undefined || options.to_line === null){ - options.to_line = -1; }else { - if(typeof options.to_line === 'string' && /\d+/.test(options.to_line)){ - options.to_line = parseInt(options.to_line); - } - if(Number.isInteger(options.to_line)){ - if(options.to_line <= 0){ - throw new Error(`Invalid Option: to_line must be a positive integer greater than 0, got ${JSON.stringify(opts.to_line)}`); - } - }else { - throw new Error(`Invalid Option: to_line must be an integer, got ${JSON.stringify(opts.to_line)}`); - } + throw new Error(`Invalid Option: to must be an integer, got ${JSON.stringify(opts.to)}`); } - this.info = { - bytes: 0, - comment_lines: 0, - empty_lines: 0, - invalid_field_length: 0, - lines: 1, - records: 0 - }; - this.options = options; - this.state = { - bomSkipped: false, - bufBytesStart: 0, - castField: fnCastField, - commenting: false, - // Current error encountered by a record - error: undefined, - enabled: options.from_line === 1, - escaping: false, - escapeIsQuote: isBuffer$1(options.escape) && isBuffer$1(options.quote) && Buffer.compare(options.escape, options.quote) === 0, - // columns can be `false`, `true`, `Array` - expectedRecordLength: Array.isArray(options.columns) ? options.columns.length : undefined, - field: new ResizeableBuffer(20), - firstLineToHeaders: fnFirstLineToHeaders, - needMoreDataSize: Math.max( - // Skip if the remaining buffer smaller than comment - options.comment !== null ? options.comment.length : 0, - // Skip if the remaining buffer can be delimiter - ...options.delimiter.map((delimiter) => delimiter.length), - // Skip if the remaining buffer can be escape sequence - options.quote !== null ? options.quote.length : 0, - ), - previousBuf: undefined, - quoting: false, - stop: false, - rawBuffer: new ResizeableBuffer(100), - record: [], - recordHasError: false, - record_length: 0, - recordDelimiterMaxLength: options.record_delimiter.length === 0 ? 2 : Math.max(...options.record_delimiter.map((v) => v.length)), - trimChars: [Buffer.from(' ', options.encoding)[0], Buffer.from('\t', options.encoding)[0]], - wasQuoting: false, - wasRowDelimiter: false - }; } - // Implementation of `Transform._transform` - _transform(buf, encoding, callback){ - if(this.state.stop === true){ - return; - } - const err = this.__parse(buf, false); - if(err !== undefined){ - this.state.stop = true; - } - callback(err); - } - // Implementation of `Transform._flush` - _flush(callback){ - if(this.state.stop === true){ - return; + // Normalize option `to_line` + if(options.to_line === undefined || options.to_line === null){ + options.to_line = -1; + }else { + if(typeof options.to_line === 'string' && /\d+/.test(options.to_line)){ + options.to_line = parseInt(options.to_line); } - const err = this.__parse(undefined, true); - callback(err); - } - // Central parser implementation - __parse(nextBuf, end){ - const {bom, comment, escape, from_line, ltrim, max_record_size, quote, raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options; - let {record_delimiter} = this.options; - const {bomSkipped, previousBuf, rawBuffer, escapeIsQuote} = this.state; - let buf; - if(previousBuf === undefined){ - if(nextBuf === undefined){ - // Handle empty string - this.push(null); - return; - }else { - buf = nextBuf; + if(Number.isInteger(options.to_line)){ + if(options.to_line <= 0){ + throw new Error(`Invalid Option: to_line must be a positive integer greater than 0, got ${JSON.stringify(opts.to_line)}`); } - }else if(previousBuf !== undefined && nextBuf === undefined){ - buf = previousBuf; }else { - buf = Buffer.concat([previousBuf, nextBuf]); - } - // Handle UTF BOM - if(bomSkipped === false){ - if(bom === false){ - this.state.bomSkipped = true; - }else if(buf.length < 3){ - // No enough data - if(end === false){ - // Wait for more data - this.state.previousBuf = buf; + throw new Error(`Invalid Option: to_line must be an integer, got ${JSON.stringify(opts.to_line)}`); + } + } + return options; + }; + + const isRecordEmpty = function(record){ + return record.every((field) => field == null || field.toString && field.toString().trim() === ''); + }; + + // white space characters + // https://en.wikipedia.org/wiki/Whitespace_character + // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions/Character_Classes#Types + // \f\n\r\t\v\u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff + const tab = 9; + const nl = 10; // \n, 0x0A in hexadecimal, 10 in decimal + const np = 12; + const cr = 13; // \r, 0x0D in hexadécimal, 13 in decimal + const space = 32; + const boms = { + // Note, the following are equals: + // Buffer.from("\ufeff") + // Buffer.from([239, 187, 191]) + // Buffer.from('EFBBBF', 'hex') + 'utf8': Buffer.from([239, 187, 191]), + // Note, the following are equals: + // Buffer.from "\ufeff", 'utf16le + // Buffer.from([255, 254]) + 'utf16le': Buffer.from([255, 254]) + }; + + const transform$1 = function(original_options = {}) { + const info = { + bytes: 0, + comment_lines: 0, + empty_lines: 0, + invalid_field_length: 0, + lines: 1, + records: 0 + }; + const options = normalize_options$1(original_options); + return { + info: info, + original_options: original_options, + options: options, + state: init_state(options), + __needMoreData: function(i, bufLen, end){ + if(end) return false; + const {quote} = this.options; + const {quoting, needMoreDataSize, recordDelimiterMaxLength} = this.state; + const numOfCharLeft = bufLen - i - 1; + const requiredLength = Math.max( + needMoreDataSize, + // Skip if the remaining buffer smaller than record delimiter + recordDelimiterMaxLength, + // Skip if the remaining buffer can be record delimiter following the closing quote + // 1 is for quote.length + quoting ? (quote.length + recordDelimiterMaxLength) : 0, + ); + return numOfCharLeft < requiredLength; + }, + // Central parser implementation + parse: function(nextBuf, end, push, close){ + const {bom, comment, escape, from_line, ltrim, max_record_size, quote, raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options; + let {record_delimiter} = this.options; + const {bomSkipped, previousBuf, rawBuffer, escapeIsQuote} = this.state; + let buf; + if(previousBuf === undefined){ + if(nextBuf === undefined){ + // Handle empty string + close(); return; + }else { + buf = nextBuf; } + }else if(previousBuf !== undefined && nextBuf === undefined){ + buf = previousBuf; }else { - for(const encoding in boms){ - if(boms[encoding].compare(buf, 0, boms[encoding].length) === 0){ - // Skip BOM - const bomLength = boms[encoding].length; - this.state.bufBytesStart += bomLength; - buf = buf.slice(bomLength); - // Renormalize original options with the new encoding - this.__normalizeOptions({...this.__originalOptions, encoding: encoding}); - break; + buf = Buffer.concat([previousBuf, nextBuf]); + } + // Handle UTF BOM + if(bomSkipped === false){ + if(bom === false){ + this.state.bomSkipped = true; + }else if(buf.length < 3){ + // No enough data + if(end === false){ + // Wait for more data + this.state.previousBuf = buf; + return; } + }else { + for(const encoding in boms){ + if(boms[encoding].compare(buf, 0, boms[encoding].length) === 0){ + // Skip BOM + const bomLength = boms[encoding].length; + this.state.bufBytesStart += bomLength; + buf = buf.slice(bomLength); + // Renormalize original options with the new encoding + this.options = normalize_options$1({...this.original_options, encoding: encoding}); + break; + } + } + this.state.bomSkipped = true; } - this.state.bomSkipped = true; - } - } - const bufLen = buf.length; - let pos; - for(pos = 0; pos < bufLen; pos++){ - // Ensure we get enough space to look ahead - // There should be a way to move this out of the loop - if(this.__needMoreData(pos, bufLen, end)){ - break; - } - if(this.state.wasRowDelimiter === true){ - this.info.lines++; - this.state.wasRowDelimiter = false; - } - if(to_line !== -1 && this.info.lines > to_line){ - this.state.stop = true; - this.push(null); - return; } - // Auto discovery of record_delimiter, unix, mac and windows supported - if(this.state.quoting === false && record_delimiter.length === 0){ - const record_delimiterCount = this.__autoDiscoverRecordDelimiter(buf, pos); - if(record_delimiterCount){ - record_delimiter = this.options.record_delimiter; + const bufLen = buf.length; + let pos; + for(pos = 0; pos < bufLen; pos++){ + // Ensure we get enough space to look ahead + // There should be a way to move this out of the loop + if(this.__needMoreData(pos, bufLen, end)){ + break; } - } - const chr = buf[pos]; - if(raw === true){ - rawBuffer.append(chr); - } - if((chr === cr || chr === nl) && this.state.wasRowDelimiter === false){ - this.state.wasRowDelimiter = true; - } - // Previous char was a valid escape char - // treat the current char as a regular char - if(this.state.escaping === true){ - this.state.escaping = false; - }else { - // Escape is only active inside quoted fields - // We are quoting, the char is an escape chr and there is a chr to escape - // if(escape !== null && this.state.quoting === true && chr === escape && pos + 1 < bufLen){ - if(escape !== null && this.state.quoting === true && this.__isEscape(buf, pos, chr) && pos + escape.length < bufLen){ - if(escapeIsQuote){ - if(this.__isQuote(buf, pos+escape.length)){ - this.state.escaping = true; - pos += escape.length - 1; - continue; - } - }else { - this.state.escaping = true; - pos += escape.length - 1; - continue; + if(this.state.wasRowDelimiter === true){ + this.info.lines++; + this.state.wasRowDelimiter = false; + } + if(to_line !== -1 && this.info.lines > to_line){ + this.state.stop = true; + close(); + return; + } + // Auto discovery of record_delimiter, unix, mac and windows supported + if(this.state.quoting === false && record_delimiter.length === 0){ + const record_delimiterCount = this.__autoDiscoverRecordDelimiter(buf, pos); + if(record_delimiterCount){ + record_delimiter = this.options.record_delimiter; } } - // Not currently escaping and chr is a quote - // TODO: need to compare bytes instead of single char - if(this.state.commenting === false && this.__isQuote(buf, pos)){ - if(this.state.quoting === true){ - const nextChr = buf[pos+quote.length]; - const isNextChrTrimable = rtrim && this.__isCharTrimable(nextChr); - const isNextChrComment = comment !== null && this.__compareBytes(comment, buf, pos+quote.length, nextChr); - const isNextChrDelimiter = this.__isDelimiter(buf, pos+quote.length, nextChr); - const isNextChrRecordDelimiter = record_delimiter.length === 0 ? this.__autoDiscoverRecordDelimiter(buf, pos+quote.length) : this.__isRecordDelimiter(nextChr, buf, pos+quote.length); - // Escape a quote - // Treat next char as a regular character - if(escape !== null && this.__isEscape(buf, pos, chr) && this.__isQuote(buf, pos + escape.length)){ + const chr = buf[pos]; + if(raw === true){ + rawBuffer.append(chr); + } + if((chr === cr || chr === nl) && this.state.wasRowDelimiter === false){ + this.state.wasRowDelimiter = true; + } + // Previous char was a valid escape char + // treat the current char as a regular char + if(this.state.escaping === true){ + this.state.escaping = false; + }else { + // Escape is only active inside quoted fields + // We are quoting, the char is an escape chr and there is a chr to escape + // if(escape !== null && this.state.quoting === true && chr === escape && pos + 1 < bufLen){ + if(escape !== null && this.state.quoting === true && this.__isEscape(buf, pos, chr) && pos + escape.length < bufLen){ + if(escapeIsQuote){ + if(this.__isQuote(buf, pos+escape.length)){ + this.state.escaping = true; + pos += escape.length - 1; + continue; + } + }else { + this.state.escaping = true; pos += escape.length - 1; - }else if(!nextChr || isNextChrDelimiter || isNextChrRecordDelimiter || isNextChrComment || isNextChrTrimable){ - this.state.quoting = false; - this.state.wasQuoting = true; - pos += quote.length - 1; continue; - }else if(relax_quotes === false){ - const err = this.__error( - new CsvError$1('CSV_INVALID_CLOSING_QUOTE', [ - 'Invalid Closing Quote:', - `got "${String.fromCharCode(nextChr)}"`, - `at line ${this.info.lines}`, - 'instead of delimiter, record delimiter, trimable character', - '(if activated) or comment', - ], this.options, this.__infoField()) - ); - if(err !== undefined) return err; - }else { - this.state.quoting = false; - this.state.wasQuoting = true; - this.state.field.prepend(quote); - pos += quote.length - 1; } - }else { - if(this.state.field.length !== 0){ - // In relax_quotes mode, treat opening quote preceded by chrs as regular - if(relax_quotes === false){ + } + // Not currently escaping and chr is a quote + // TODO: need to compare bytes instead of single char + if(this.state.commenting === false && this.__isQuote(buf, pos)){ + if(this.state.quoting === true){ + const nextChr = buf[pos+quote.length]; + const isNextChrTrimable = rtrim && this.__isCharTrimable(nextChr); + const isNextChrComment = comment !== null && this.__compareBytes(comment, buf, pos+quote.length, nextChr); + const isNextChrDelimiter = this.__isDelimiter(buf, pos+quote.length, nextChr); + const isNextChrRecordDelimiter = record_delimiter.length === 0 ? this.__autoDiscoverRecordDelimiter(buf, pos+quote.length) : this.__isRecordDelimiter(nextChr, buf, pos+quote.length); + // Escape a quote + // Treat next char as a regular character + if(escape !== null && this.__isEscape(buf, pos, chr) && this.__isQuote(buf, pos + escape.length)){ + pos += escape.length - 1; + }else if(!nextChr || isNextChrDelimiter || isNextChrRecordDelimiter || isNextChrComment || isNextChrTrimable){ + this.state.quoting = false; + this.state.wasQuoting = true; + pos += quote.length - 1; + continue; + }else if(relax_quotes === false){ const err = this.__error( - new CsvError$1('INVALID_OPENING_QUOTE', [ - 'Invalid Opening Quote:', - `a quote is found inside a field at line ${this.info.lines}`, - ], this.options, this.__infoField(), { - field: this.state.field, - }) + new CsvError$1('CSV_INVALID_CLOSING_QUOTE', [ + 'Invalid Closing Quote:', + `got "${String.fromCharCode(nextChr)}"`, + `at line ${this.info.lines}`, + 'instead of delimiter, record delimiter, trimable character', + '(if activated) or comment', + ], this.options, this.__infoField()) ); if(err !== undefined) return err; + }else { + this.state.quoting = false; + this.state.wasQuoting = true; + this.state.field.prepend(quote); + pos += quote.length - 1; } }else { - this.state.quoting = true; - pos += quote.length - 1; - continue; - } - } - } - if(this.state.quoting === false){ - const recordDelimiterLength = this.__isRecordDelimiter(chr, buf, pos); - if(recordDelimiterLength !== 0){ - // Do not emit comments which take a full line - const skipCommentLine = this.state.commenting && (this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0); - if(skipCommentLine){ - this.info.comment_lines++; - // Skip full comment line - }else { - // Activate records emition if above from_line - if(this.state.enabled === false && this.info.lines + (this.state.wasRowDelimiter === true ? 1: 0) >= from_line){ - this.state.enabled = true; - this.__resetField(); - this.__resetRecord(); - pos += recordDelimiterLength - 1; + if(this.state.field.length !== 0){ + // In relax_quotes mode, treat opening quote preceded by chrs as regular + if(relax_quotes === false){ + const err = this.__error( + new CsvError$1('INVALID_OPENING_QUOTE', [ + 'Invalid Opening Quote:', + `a quote is found inside a field at line ${this.info.lines}`, + ], this.options, this.__infoField(), { + field: this.state.field, + }) + ); + if(err !== undefined) return err; + } + }else { + this.state.quoting = true; + pos += quote.length - 1; continue; } - // Skip if line is empty and skip_empty_lines activated - if(skip_empty_lines === true && this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0){ - this.info.empty_lines++; - pos += recordDelimiterLength - 1; - continue; + } + } + if(this.state.quoting === false){ + const recordDelimiterLength = this.__isRecordDelimiter(chr, buf, pos); + if(recordDelimiterLength !== 0){ + // Do not emit comments which take a full line + const skipCommentLine = this.state.commenting && (this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0); + if(skipCommentLine){ + this.info.comment_lines++; + // Skip full comment line + }else { + // Activate records emition if above from_line + if(this.state.enabled === false && this.info.lines + (this.state.wasRowDelimiter === true ? 1: 0) >= from_line){ + this.state.enabled = true; + this.__resetField(); + this.__resetRecord(); + pos += recordDelimiterLength - 1; + continue; + } + // Skip if line is empty and skip_empty_lines activated + if(skip_empty_lines === true && this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0){ + this.info.empty_lines++; + pos += recordDelimiterLength - 1; + continue; + } + this.info.bytes = this.state.bufBytesStart + pos; + const errField = this.__onField(); + if(errField !== undefined) return errField; + this.info.bytes = this.state.bufBytesStart + pos + recordDelimiterLength; + const errRecord = this.__onRecord(push); + if(errRecord !== undefined) return errRecord; + if(to !== -1 && this.info.records >= to){ + this.state.stop = true; + close(); + return; + } } + this.state.commenting = false; + pos += recordDelimiterLength - 1; + continue; + } + if(this.state.commenting){ + continue; + } + const commentCount = comment === null ? 0 : this.__compareBytes(comment, buf, pos, chr); + if(commentCount !== 0){ + this.state.commenting = true; + continue; + } + const delimiterLength = this.__isDelimiter(buf, pos, chr); + if(delimiterLength !== 0){ this.info.bytes = this.state.bufBytesStart + pos; const errField = this.__onField(); if(errField !== undefined) return errField; - this.info.bytes = this.state.bufBytesStart + pos + recordDelimiterLength; - const errRecord = this.__onRecord(); - if(errRecord !== undefined) return errRecord; - if(to !== -1 && this.info.records >= to){ - this.state.stop = true; - this.push(null); - return; - } + pos += delimiterLength - 1; + continue; } - this.state.commenting = false; - pos += recordDelimiterLength - 1; - continue; - } - if(this.state.commenting){ - continue; - } - const commentCount = comment === null ? 0 : this.__compareBytes(comment, buf, pos, chr); - if(commentCount !== 0){ - this.state.commenting = true; - continue; } - const delimiterLength = this.__isDelimiter(buf, pos, chr); - if(delimiterLength !== 0){ - this.info.bytes = this.state.bufBytesStart + pos; - const errField = this.__onField(); - if(errField !== undefined) return errField; - pos += delimiterLength - 1; - continue; + } + if(this.state.commenting === false){ + if(max_record_size !== 0 && this.state.record_length + this.state.field.length > max_record_size){ + const err = this.__error( + new CsvError$1('CSV_MAX_RECORD_SIZE', [ + 'Max Record Size:', + 'record exceed the maximum number of tolerated bytes', + `of ${max_record_size}`, + `at line ${this.info.lines}`, + ], this.options, this.__infoField()) + ); + if(err !== undefined) return err; } } - } - if(this.state.commenting === false){ - if(max_record_size !== 0 && this.state.record_length + this.state.field.length > max_record_size){ + const lappend = ltrim === false || this.state.quoting === true || this.state.field.length !== 0 || !this.__isCharTrimable(chr); + // rtrim in non quoting is handle in __onField + const rappend = rtrim === false || this.state.wasQuoting === false; + if(lappend === true && rappend === true){ + this.state.field.append(chr); + }else if(rtrim === true && !this.__isCharTrimable(chr)){ const err = this.__error( - new CsvError$1('CSV_MAX_RECORD_SIZE', [ - 'Max Record Size:', - 'record exceed the maximum number of tolerated bytes', - `of ${max_record_size}`, + new CsvError$1('CSV_NON_TRIMABLE_CHAR_AFTER_CLOSING_QUOTE', [ + 'Invalid Closing Quote:', + 'found non trimable byte after quote', `at line ${this.info.lines}`, ], this.options, this.__infoField()) ); if(err !== undefined) return err; } } - const lappend = ltrim === false || this.state.quoting === true || this.state.field.length !== 0 || !this.__isCharTrimable(chr); - // rtrim in non quoting is handle in __onField - const rappend = rtrim === false || this.state.wasQuoting === false; - if(lappend === true && rappend === true){ - this.state.field.append(chr); - }else if(rtrim === true && !this.__isCharTrimable(chr)){ - const err = this.__error( - new CsvError$1('CSV_NON_TRIMABLE_CHAR_AFTER_CLOSING_QUOTE', [ - 'Invalid Closing Quote:', - 'found non trimable byte after quote', - `at line ${this.info.lines}`, - ], this.options, this.__infoField()) - ); - if(err !== undefined) return err; - } - } - if(end === true){ - // Ensure we are not ending in a quoting state - if(this.state.quoting === true){ - const err = this.__error( - new CsvError$1('CSV_QUOTE_NOT_CLOSED', [ - 'Quote Not Closed:', - `the parsing is finished with an opening quote at line ${this.info.lines}`, - ], this.options, this.__infoField()) - ); - if(err !== undefined) return err; + if(end === true){ + // Ensure we are not ending in a quoting state + if(this.state.quoting === true){ + const err = this.__error( + new CsvError$1('CSV_QUOTE_NOT_CLOSED', [ + 'Quote Not Closed:', + `the parsing is finished with an opening quote at line ${this.info.lines}`, + ], this.options, this.__infoField()) + ); + if(err !== undefined) return err; + }else { + // Skip last line if it has no characters + if(this.state.wasQuoting === true || this.state.record.length !== 0 || this.state.field.length !== 0){ + this.info.bytes = this.state.bufBytesStart + pos; + const errField = this.__onField(); + if(errField !== undefined) return errField; + const errRecord = this.__onRecord(push); + if(errRecord !== undefined) return errRecord; + }else if(this.state.wasRowDelimiter === true){ + this.info.empty_lines++; + }else if(this.state.commenting === true){ + this.info.comment_lines++; + } + } }else { - // Skip last line if it has no characters - if(this.state.wasQuoting === true || this.state.record.length !== 0 || this.state.field.length !== 0){ - this.info.bytes = this.state.bufBytesStart + pos; - const errField = this.__onField(); - if(errField !== undefined) return errField; - const errRecord = this.__onRecord(); - if(errRecord !== undefined) return errRecord; - }else if(this.state.wasRowDelimiter === true){ - this.info.empty_lines++; - }else if(this.state.commenting === true){ - this.info.comment_lines++; + this.state.bufBytesStart += pos; + this.state.previousBuf = buf.slice(pos); + } + if(this.state.wasRowDelimiter === true){ + this.info.lines++; + this.state.wasRowDelimiter = false; + } + }, + __onRecord: function(push){ + const {columns, group_columns_by_name, encoding, info, from, relax_column_count, relax_column_count_less, relax_column_count_more, raw, skip_records_with_empty_values} = this.options; + const {enabled, record} = this.state; + if(enabled === false){ + return this.__resetRecord(); + } + // Convert the first line into column names + const recordLength = record.length; + if(columns === true){ + if(skip_records_with_empty_values === true && isRecordEmpty(record)){ + this.__resetRecord(); + return; + } + return this.__firstLineToColumns(record); + } + if(columns === false && this.info.records === 0){ + this.state.expectedRecordLength = recordLength; + } + if(recordLength !== this.state.expectedRecordLength){ + const err = columns === false ? + new CsvError$1('CSV_RECORD_INCONSISTENT_FIELDS_LENGTH', [ + 'Invalid Record Length:', + `expect ${this.state.expectedRecordLength},`, + `got ${recordLength} on line ${this.info.lines}`, + ], this.options, this.__infoField(), { + record: record, + }) + : + new CsvError$1('CSV_RECORD_INCONSISTENT_COLUMNS', [ + 'Invalid Record Length:', + `columns length is ${columns.length},`, // rename columns + `got ${recordLength} on line ${this.info.lines}`, + ], this.options, this.__infoField(), { + record: record, + }); + if(relax_column_count === true || + (relax_column_count_less === true && recordLength < this.state.expectedRecordLength) || + (relax_column_count_more === true && recordLength > this.state.expectedRecordLength)){ + this.info.invalid_field_length++; + this.state.error = err; + // Error is undefined with skip_records_with_error + }else { + const finalErr = this.__error(err); + if(finalErr) return finalErr; } } - }else { - this.state.bufBytesStart += pos; - this.state.previousBuf = buf.slice(pos); - } - if(this.state.wasRowDelimiter === true){ - this.info.lines++; - this.state.wasRowDelimiter = false; - } - } - __onRecord(){ - const {columns, group_columns_by_name, encoding, info, from, relax_column_count, relax_column_count_less, relax_column_count_more, raw, skip_records_with_empty_values} = this.options; - const {enabled, record} = this.state; - if(enabled === false){ - return this.__resetRecord(); - } - // Convert the first line into column names - const recordLength = record.length; - if(columns === true){ if(skip_records_with_empty_values === true && isRecordEmpty(record)){ this.__resetRecord(); return; } - return this.__firstLineToColumns(record); - } - if(columns === false && this.info.records === 0){ - this.state.expectedRecordLength = recordLength; - } - if(recordLength !== this.state.expectedRecordLength){ - const err = columns === false ? - new CsvError$1('CSV_RECORD_INCONSISTENT_FIELDS_LENGTH', [ - 'Invalid Record Length:', - `expect ${this.state.expectedRecordLength},`, - `got ${recordLength} on line ${this.info.lines}`, - ], this.options, this.__infoField(), { - record: record, - }) - : - new CsvError$1('CSV_RECORD_INCONSISTENT_COLUMNS', [ - 'Invalid Record Length:', - `columns length is ${columns.length},`, // rename columns - `got ${recordLength} on line ${this.info.lines}`, - ], this.options, this.__infoField(), { - record: record, - }); - if(relax_column_count === true || - (relax_column_count_less === true && recordLength < this.state.expectedRecordLength) || - (relax_column_count_more === true && recordLength > this.state.expectedRecordLength)){ - this.info.invalid_field_length++; - this.state.error = err; - // Error is undefined with skip_records_with_error - }else { - const finalErr = this.__error(err); - if(finalErr) return finalErr; + if(this.state.recordHasError === true){ + this.__resetRecord(); + this.state.recordHasError = false; + return; } - } - if(skip_records_with_empty_values === true && isRecordEmpty(record)){ - this.__resetRecord(); - return; - } - if(this.state.recordHasError === true){ - this.__resetRecord(); - this.state.recordHasError = false; - return; - } - this.info.records++; - if(from === 1 || this.info.records >= from){ - const {objname} = this.options; - // With columns, records are object - if(columns !== false){ - const obj = {}; - // Transform record array to an object - for(let i = 0, l = record.length; i < l; i++){ - if(columns[i] === undefined || columns[i].disabled) continue; - // Turn duplicate columns into an array - if (group_columns_by_name === true && obj[columns[i].name] !== undefined) { - if (Array.isArray(obj[columns[i].name])) { - obj[columns[i].name] = obj[columns[i].name].concat(record[i]); + this.info.records++; + if(from === 1 || this.info.records >= from){ + const {objname} = this.options; + // With columns, records are object + if(columns !== false){ + const obj = {}; + // Transform record array to an object + for(let i = 0, l = record.length; i < l; i++){ + if(columns[i] === undefined || columns[i].disabled) continue; + // Turn duplicate columns into an array + if (group_columns_by_name === true && obj[columns[i].name] !== undefined) { + if (Array.isArray(obj[columns[i].name])) { + obj[columns[i].name] = obj[columns[i].name].concat(record[i]); + } else { + obj[columns[i].name] = [obj[columns[i].name], record[i]]; + } } else { - obj[columns[i].name] = [obj[columns[i].name], record[i]]; + obj[columns[i].name] = record[i]; } - } else { - obj[columns[i].name] = record[i]; } - } - // Without objname (default) - if(raw === true || info === true){ - const extRecord = Object.assign( - {record: obj}, - (raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}), - (info === true ? {info: this.__infoRecord()}: {}) - ); - const err = this.__push( - objname === undefined ? extRecord : [obj[objname], extRecord] - ); - if(err){ - return err; - } - }else { - const err = this.__push( - objname === undefined ? obj : [obj[objname], obj] - ); - if(err){ - return err; - } - } - // Without columns, records are array - }else { - if(raw === true || info === true){ - const extRecord = Object.assign( - {record: record}, - raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}, - info === true ? {info: this.__infoRecord()}: {} - ); - const err = this.__push( - objname === undefined ? extRecord : [record[objname], extRecord] - ); - if(err){ - return err; + // Without objname (default) + if(raw === true || info === true){ + const extRecord = Object.assign( + {record: obj}, + (raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}), + (info === true ? {info: this.__infoRecord()}: {}) + ); + const err = this.__push( + objname === undefined ? extRecord : [obj[objname], extRecord] + , push); + if(err){ + return err; + } + }else { + const err = this.__push( + objname === undefined ? obj : [obj[objname], obj] + , push); + if(err){ + return err; + } } + // Without columns, records are array }else { - const err = this.__push( - objname === undefined ? record : [record[objname], record] - ); - if(err){ - return err; + if(raw === true || info === true){ + const extRecord = Object.assign( + {record: record}, + raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}, + info === true ? {info: this.__infoRecord()}: {} + ); + const err = this.__push( + objname === undefined ? extRecord : [record[objname], extRecord] + , push); + if(err){ + return err; + } + }else { + const err = this.__push( + objname === undefined ? record : [record[objname], record] + , push); + if(err){ + return err; + } } } } - } - this.__resetRecord(); - } - __firstLineToColumns(record){ - const {firstLineToHeaders} = this.state; - try{ - const headers = firstLineToHeaders === undefined ? record : firstLineToHeaders.call(null, record); - if(!Array.isArray(headers)){ - return this.__error( - new CsvError$1('CSV_INVALID_COLUMN_MAPPING', [ - 'Invalid Column Mapping:', - 'expect an array from column function,', - `got ${JSON.stringify(headers)}` - ], this.options, this.__infoField(), { - headers: headers, - }) - ); - } - const normalizedHeaders = normalizeColumnsArray(headers); - this.state.expectedRecordLength = normalizedHeaders.length; - this.options.columns = normalizedHeaders; this.__resetRecord(); - return; - }catch(err){ - return err; - } - } - __resetRecord(){ - if(this.options.raw === true){ - this.state.rawBuffer.reset(); - } - this.state.error = undefined; - this.state.record = []; - this.state.record_length = 0; - } - __onField(){ - const {cast, encoding, rtrim, max_record_size} = this.options; - const {enabled, wasQuoting} = this.state; - // Short circuit for the from_line options - if(enabled === false){ - return this.__resetField(); - } - let field = this.state.field.toString(encoding); - if(rtrim === true && wasQuoting === false){ - field = field.trimRight(); - } - if(cast === true){ - const [err, f] = this.__cast(field); - if(err !== undefined) return err; - field = f; - } - this.state.record.push(field); - // Increment record length if record size must not exceed a limit - if(max_record_size !== 0 && typeof field === 'string'){ - this.state.record_length += field.length; - } - this.__resetField(); - } - __resetField(){ - this.state.field.reset(); - this.state.wasQuoting = false; - } - __push(record){ - const {on_record} = this.options; - if(on_record !== undefined){ - const info = this.__infoRecord(); + }, + __firstLineToColumns: function(record){ + const {firstLineToHeaders} = this.state; try{ - record = on_record.call(null, record, info); + const headers = firstLineToHeaders === undefined ? record : firstLineToHeaders.call(null, record); + if(!Array.isArray(headers)){ + return this.__error( + new CsvError$1('CSV_INVALID_COLUMN_MAPPING', [ + 'Invalid Column Mapping:', + 'expect an array from column function,', + `got ${JSON.stringify(headers)}` + ], this.options, this.__infoField(), { + headers: headers, + }) + ); + } + const normalizedHeaders = normalize_columns_array(headers); + this.state.expectedRecordLength = normalizedHeaders.length; + this.options.columns = normalizedHeaders; + this.__resetRecord(); + return; }catch(err){ return err; } - if(record === undefined || record === null){ return; } - } - this.push(record); - } - // Return a tuple with the error and the casted value - __cast(field){ - const {columns, relax_column_count} = this.options; - const isColumns = Array.isArray(columns); - // Dont loose time calling cast - // because the final record is an object - // and this field can't be associated to a key present in columns - if(isColumns === true && relax_column_count && this.options.columns.length <= this.state.record.length){ - return [undefined, undefined]; - } - if(this.state.castField !== null){ - try{ + }, + __resetRecord: function(){ + if(this.options.raw === true){ + this.state.rawBuffer.reset(); + } + this.state.error = undefined; + this.state.record = []; + this.state.record_length = 0; + }, + __onField: function(){ + const {cast, encoding, rtrim, max_record_size} = this.options; + const {enabled, wasQuoting} = this.state; + // Short circuit for the from_line options + if(enabled === false){ + return this.__resetField(); + } + let field = this.state.field.toString(encoding); + if(rtrim === true && wasQuoting === false){ + field = field.trimRight(); + } + if(cast === true){ + const [err, f] = this.__cast(field); + if(err !== undefined) return err; + field = f; + } + this.state.record.push(field); + // Increment record length if record size must not exceed a limit + if(max_record_size !== 0 && typeof field === 'string'){ + this.state.record_length += field.length; + } + this.__resetField(); + }, + __resetField: function(){ + this.state.field.reset(); + this.state.wasQuoting = false; + }, + __push: function(record, push){ + const {on_record} = this.options; + if(on_record !== undefined){ + const info = this.__infoRecord(); + try{ + record = on_record.call(null, record, info); + }catch(err){ + return err; + } + if(record === undefined || record === null){ return; } + } + push(record); + }, + // Return a tuple with the error and the casted value + __cast: function(field){ + const {columns, relax_column_count} = this.options; + const isColumns = Array.isArray(columns); + // Dont loose time calling cast + // because the final record is an object + // and this field can't be associated to a key present in columns + if(isColumns === true && relax_column_count && this.options.columns.length <= this.state.record.length){ + return [undefined, undefined]; + } + if(this.state.castField !== null){ + try{ + const info = this.__infoField(); + return [undefined, this.state.castField.call(null, field, info)]; + }catch(err){ + return [err]; + } + } + if(this.__isFloat(field)){ + return [undefined, parseFloat(field)]; + }else if(this.options.cast_date !== false){ const info = this.__infoField(); - return [undefined, this.state.castField.call(null, field, info)]; - }catch(err){ - return [err]; + return [undefined, this.options.cast_date.call(null, field, info)]; } - } - if(this.__isFloat(field)){ - return [undefined, parseFloat(field)]; - }else if(this.options.cast_date !== false){ - const info = this.__infoField(); - return [undefined, this.options.cast_date.call(null, field, info)]; - } - return [undefined, field]; - } - // Helper to test if a character is a space or a line delimiter - __isCharTrimable(chr){ - return chr === space || chr === tab || chr === cr || chr === nl || chr === np; - } - // Keep it in case we implement the `cast_int` option - // __isInt(value){ - // // return Number.isInteger(parseInt(value)) - // // return !isNaN( parseInt( obj ) ); - // return /^(\-|\+)?[1-9][0-9]*$/.test(value) - // } - __isFloat(value){ - return (value - parseFloat(value) + 1) >= 0; // Borrowed from jquery - } - __compareBytes(sourceBuf, targetBuf, targetPos, firstByte){ - if(sourceBuf[0] !== firstByte) return 0; - const sourceLength = sourceBuf.length; - for(let i = 1; i < sourceLength; i++){ - if(sourceBuf[i] !== targetBuf[targetPos+i]) return 0; - } - return sourceLength; - } - __needMoreData(i, bufLen, end){ - if(end) return false; - const {quote} = this.options; - const {quoting, needMoreDataSize, recordDelimiterMaxLength} = this.state; - const numOfCharLeft = bufLen - i - 1; - const requiredLength = Math.max( - needMoreDataSize, - // Skip if the remaining buffer smaller than record delimiter - recordDelimiterMaxLength, - // Skip if the remaining buffer can be record delimiter following the closing quote - // 1 is for quote.length - quoting ? (quote.length + recordDelimiterMaxLength) : 0, - ); - return numOfCharLeft < requiredLength; - } - __isDelimiter(buf, pos, chr){ - const {delimiter, ignore_last_delimiters} = this.options; - if(ignore_last_delimiters === true && this.state.record.length === this.options.columns.length - 1){ - return 0; - }else if(ignore_last_delimiters !== false && typeof ignore_last_delimiters === 'number' && this.state.record.length === ignore_last_delimiters - 1){ - return 0; - } - loop1: for(let i = 0; i < delimiter.length; i++){ - const del = delimiter[i]; - if(del[0] === chr){ - for(let j = 1; j < del.length; j++){ - if(del[j] !== buf[pos+j]) continue loop1; + return [undefined, field]; + }, + // Helper to test if a character is a space or a line delimiter + __isCharTrimable: function(chr){ + return chr === space || chr === tab || chr === cr || chr === nl || chr === np; + }, + // Keep it in case we implement the `cast_int` option + // __isInt(value){ + // // return Number.isInteger(parseInt(value)) + // // return !isNaN( parseInt( obj ) ); + // return /^(\-|\+)?[1-9][0-9]*$/.test(value) + // } + __isFloat: function(value){ + return (value - parseFloat(value) + 1) >= 0; // Borrowed from jquery + }, + __compareBytes: function(sourceBuf, targetBuf, targetPos, firstByte){ + if(sourceBuf[0] !== firstByte) return 0; + const sourceLength = sourceBuf.length; + for(let i = 1; i < sourceLength; i++){ + if(sourceBuf[i] !== targetBuf[targetPos+i]) return 0; + } + return sourceLength; + }, + __isDelimiter: function(buf, pos, chr){ + const {delimiter, ignore_last_delimiters} = this.options; + if(ignore_last_delimiters === true && this.state.record.length === this.options.columns.length - 1){ + return 0; + }else if(ignore_last_delimiters !== false && typeof ignore_last_delimiters === 'number' && this.state.record.length === ignore_last_delimiters - 1){ + return 0; + } + loop1: for(let i = 0; i < delimiter.length; i++){ + const del = delimiter[i]; + if(del[0] === chr){ + for(let j = 1; j < del.length; j++){ + if(del[j] !== buf[pos+j]) continue loop1; + } + return del.length; } - return del.length; } - } - return 0; - } - __isRecordDelimiter(chr, buf, pos){ - const {record_delimiter} = this.options; - const recordDelimiterLength = record_delimiter.length; - loop1: for(let i = 0; i < recordDelimiterLength; i++){ - const rd = record_delimiter[i]; - const rdLength = rd.length; - if(rd[0] !== chr){ - continue; + return 0; + }, + __isRecordDelimiter: function(chr, buf, pos){ + const {record_delimiter} = this.options; + const recordDelimiterLength = record_delimiter.length; + loop1: for(let i = 0; i < recordDelimiterLength; i++){ + const rd = record_delimiter[i]; + const rdLength = rd.length; + if(rd[0] !== chr){ + continue; + } + for(let j = 1; j < rdLength; j++){ + if(rd[j] !== buf[pos+j]){ + continue loop1; + } + } + return rd.length; } - for(let j = 1; j < rdLength; j++){ - if(rd[j] !== buf[pos+j]){ - continue loop1; + return 0; + }, + __isEscape: function(buf, pos, chr){ + const {escape} = this.options; + if(escape === null) return false; + const l = escape.length; + if(escape[0] === chr){ + for(let i = 0; i < l; i++){ + if(escape[i] !== buf[pos+i]){ + return false; + } } + return true; } - return rd.length; - } - return 0; - } - __isEscape(buf, pos, chr){ - const {escape} = this.options; - if(escape === null) return false; - const l = escape.length; - if(escape[0] === chr){ + return false; + }, + __isQuote: function(buf, pos){ + const {quote} = this.options; + if(quote === null) return false; + const l = quote.length; for(let i = 0; i < l; i++){ - if(escape[i] !== buf[pos+i]){ + if(quote[i] !== buf[pos+i]){ return false; } } return true; - } - return false; - } - __isQuote(buf, pos){ - const {quote} = this.options; - if(quote === null) return false; - const l = quote.length; - for(let i = 0; i < l; i++){ - if(quote[i] !== buf[pos+i]){ - return false; - } - } - return true; - } - __autoDiscoverRecordDelimiter(buf, pos){ - const {encoding} = this.options; - const chr = buf[pos]; - if(chr === cr){ - if(buf[pos+1] === nl){ - this.options.record_delimiter.push(Buffer.from('\r\n', encoding)); - this.state.recordDelimiterMaxLength = 2; - return 2; - }else { - this.options.record_delimiter.push(Buffer.from('\r', encoding)); + }, + __autoDiscoverRecordDelimiter: function(buf, pos){ + const {encoding} = this.options; + const chr = buf[pos]; + if(chr === cr){ + if(buf[pos+1] === nl){ + this.options.record_delimiter.push(Buffer.from('\r\n', encoding)); + this.state.recordDelimiterMaxLength = 2; + return 2; + }else { + this.options.record_delimiter.push(Buffer.from('\r', encoding)); + this.state.recordDelimiterMaxLength = 1; + return 1; + } + }else if(chr === nl){ + this.options.record_delimiter.push(Buffer.from('\n', encoding)); this.state.recordDelimiterMaxLength = 1; return 1; } - }else if(chr === nl){ - this.options.record_delimiter.push(Buffer.from('\n', encoding)); - this.state.recordDelimiterMaxLength = 1; - return 1; - } - return 0; - } - __error(msg){ - const {encoding, raw, skip_records_with_error} = this.options; - const err = typeof msg === 'string' ? new Error(msg) : msg; - if(skip_records_with_error){ - this.state.recordHasError = true; - this.emit('skip', err, raw ? this.state.rawBuffer.toString(encoding) : undefined); - return undefined; - }else { - return err; + return 0; + }, + __error: function(msg){ + const {encoding, raw, skip_records_with_error} = this.options; + const err = typeof msg === 'string' ? new Error(msg) : msg; + if(skip_records_with_error){ + this.state.recordHasError = true; + if(this.options.on_skip !== undefined){ + this.options.on_skip(err, raw ? this.state.rawBuffer.toString(encoding) : undefined); + } + // this.emit('skip', err, raw ? this.state.rawBuffer.toString(encoding) : undefined); + return undefined; + }else { + return err; + } + }, + __infoDataSet: function(){ + return { + ...this.info, + columns: this.options.columns + }; + }, + __infoRecord: function(){ + const {columns, raw, encoding} = this.options; + return { + ...this.__infoDataSet(), + error: this.state.error, + header: columns === true, + index: this.state.record.length, + raw: raw ? this.state.rawBuffer.toString(encoding) : undefined + }; + }, + __infoField: function(){ + const {columns} = this.options; + const isColumns = Array.isArray(columns); + return { + ...this.__infoRecord(), + column: isColumns === true ? + (columns.length > this.state.record.length ? + columns[this.state.record.length].name : + null + ) : + this.state.record.length, + quoting: this.state.wasQuoting, + }; } - } - __infoDataSet(){ - return { - ...this.info, - columns: this.options.columns - }; - } - __infoRecord(){ - const {columns, raw, encoding} = this.options; - return { - ...this.__infoDataSet(), - error: this.state.error, - header: columns === true, - index: this.state.record.length, - raw: raw ? this.state.rawBuffer.toString(encoding) : undefined - }; - } - __infoField(){ - const {columns} = this.options; - const isColumns = Array.isArray(columns); - return { - ...this.__infoRecord(), - column: isColumns === true ? - (columns.length > this.state.record.length ? - columns[this.state.record.length].name : - null - ) : - this.state.record.length, - quoting: this.state.wasQuoting, - }; - } - } + }; + }; - const parse = function(data, options={}){ + const parse = function(data, opts={}){ if(typeof data === 'string'){ data = Buffer.from(data); } - const records = options && options.objname ? {} : []; - const parser = new Parser(options); - parser.push = function(record){ - if(record === null){ - return; - } - if(options.objname === undefined) + const records = opts && opts.objname ? {} : []; + const parser = transform$1(opts); + const push = (record) => { + if(parser.options.objname === undefined) records.push(record); else { records[record[0]] = record[1]; } }; - const err1 = parser.__parse(data, false); + const close = () => {}; + const err1 = parser.parse(data, false, push, close); if(err1 !== undefined) throw err1; - const err2 = parser.__parse(undefined, true); + const err2 = parser.parse(undefined, true, push, close); if(err2 !== undefined) throw err2; return records; }; - const bom_utf8 = Buffer.from([239, 187, 191]); - - class CsvError extends Error { - constructor(code, message, ...contexts) { - if(Array.isArray(message)) message = message.join(' '); - super(message); - if(Error.captureStackTrace !== undefined){ - Error.captureStackTrace(this, CsvError); - } - this.code = code; - for(const context of contexts){ - for(const key in context){ - const value = context[key]; - this[key] = isBuffer$1(value) ? value.toString() : value == null ? value : JSON.parse(JSON.stringify(value)); - } - } - } - } - - const isObject = function(obj){ - return typeof obj === 'object' && obj !== null && ! Array.isArray(obj); - }; - - const underscore = function(str){ - return str.replace(/([A-Z])/g, function(_, match){ - return '_' + match.toLowerCase(); - }); - }; - // Lodash implementation of `get` const charCodeOfDot = '.'.charCodeAt(0); @@ -6656,455 +6648,473 @@ return (index && index === length) ? object : undefined; }; - class Stringifier extends Transform { - constructor(opts = {}){ - super({...{writableObjectMode: true}, ...opts}); - const options = {}; - let err; - // Merge with user options - for(const opt in opts){ - options[underscore(opt)] = opts[opt]; - } - if((err = this.normalize(options)) !== undefined) throw err; - switch(options.record_delimiter){ - case 'auto': - options.record_delimiter = null; - break; - case 'unix': - options.record_delimiter = "\n"; - break; - case 'mac': - options.record_delimiter = "\r"; - break; - case 'windows': - options.record_delimiter = "\r\n"; - break; - case 'ascii': - options.record_delimiter = "\u001e"; - break; - case 'unicode': - options.record_delimiter = "\u2028"; - break; - } - // Expose options - this.options = options; - // Internal state - this.state = { - stop: false - }; - // Information - this.info = { - records: 0 - }; + const is_object = function(obj){ + return typeof obj === 'object' && obj !== null && ! Array.isArray(obj); + }; + + const normalize_columns = function(columns){ + if(columns === undefined || columns === null){ + return [undefined, undefined]; } - normalize(options){ - // Normalize option `bom` - if(options.bom === undefined || options.bom === null || options.bom === false){ - options.bom = false; - }else if(options.bom !== true){ - return new CsvError('CSV_OPTION_BOOLEAN_INVALID_TYPE', [ - 'option `bom` is optional and must be a boolean value,', - `got ${JSON.stringify(options.bom)}` - ]); - } - // Normalize option `delimiter` - if(options.delimiter === undefined || options.delimiter === null){ - options.delimiter = ','; - }else if(isBuffer$1(options.delimiter)){ - options.delimiter = options.delimiter.toString(); - }else if(typeof options.delimiter !== 'string'){ - return new CsvError('CSV_OPTION_DELIMITER_INVALID_TYPE', [ - 'option `delimiter` must be a buffer or a string,', - `got ${JSON.stringify(options.delimiter)}` - ]); - } - // Normalize option `quote` - if(options.quote === undefined || options.quote === null){ - options.quote = '"'; - }else if(options.quote === true){ - options.quote = '"'; - }else if(options.quote === false){ - options.quote = ''; - }else if (isBuffer$1(options.quote)){ - options.quote = options.quote.toString(); - }else if(typeof options.quote !== 'string'){ - return new CsvError('CSV_OPTION_QUOTE_INVALID_TYPE', [ - 'option `quote` must be a boolean, a buffer or a string,', - `got ${JSON.stringify(options.quote)}` - ]); + if(typeof columns !== 'object'){ + return [Error('Invalid option "columns": expect an array or an object')]; + } + if(!Array.isArray(columns)){ + const newcolumns = []; + for(const k in columns){ + newcolumns.push({ + key: k, + header: columns[k] + }); } - // Normalize option `quoted` - if(options.quoted === undefined || options.quoted === null){ - options.quoted = false; - } - // Normalize option `quoted_empty` - if(options.quoted_empty === undefined || options.quoted_empty === null){ - options.quoted_empty = undefined; - } - // Normalize option `quoted_match` - if(options.quoted_match === undefined || options.quoted_match === null || options.quoted_match === false){ - options.quoted_match = null; - }else if(!Array.isArray(options.quoted_match)){ - options.quoted_match = [options.quoted_match]; - } - if(options.quoted_match){ - for(const quoted_match of options.quoted_match){ - const isString = typeof quoted_match === 'string'; - const isRegExp = quoted_match instanceof RegExp; - if(!isString && !isRegExp){ - return Error(`Invalid Option: quoted_match must be a string or a regex, got ${JSON.stringify(quoted_match)}`); + columns = newcolumns; + }else { + const newcolumns = []; + for(const column of columns){ + if(typeof column === 'string'){ + newcolumns.push({ + key: column, + header: column + }); + }else if(typeof column === 'object' && column !== null && !Array.isArray(column)){ + if(!column.key){ + return [Error('Invalid column definition: property "key" is required')]; } + if(column.header === undefined){ + column.header = column.key; + } + newcolumns.push(column); + }else { + return [Error('Invalid column definition: expect a string or an object')]; } } - // Normalize option `quoted_string` - if(options.quoted_string === undefined || options.quoted_string === null){ - options.quoted_string = false; - } - // Normalize option `eof` - if(options.eof === undefined || options.eof === null){ - options.eof = true; - } - // Normalize option `escape` - if(options.escape === undefined || options.escape === null){ - options.escape = '"'; - }else if(isBuffer$1(options.escape)){ - options.escape = options.escape.toString(); - }else if(typeof options.escape !== 'string'){ - return Error(`Invalid Option: escape must be a buffer or a string, got ${JSON.stringify(options.escape)}`); - } - if (options.escape.length > 1){ - return Error(`Invalid Option: escape must be one character, got ${options.escape.length} characters`); - } - // Normalize option `header` - if(options.header === undefined || options.header === null){ - options.header = false; - } - // Normalize option `columns` - const [err, columns] = this.normalize_columns(options.columns); - if(err) return err; - options.columns = columns; - // Normalize option `quoted` - if(options.quoted === undefined || options.quoted === null){ - options.quoted = false; - } - // Normalize option `cast` - if(options.cast === undefined || options.cast === null){ - options.cast = {}; - } - // Normalize option cast.bigint - if(options.cast.bigint === undefined || options.cast.bigint === null){ - // Cast boolean to string by default - options.cast.bigint = value => '' + value; - } - // Normalize option cast.boolean - if(options.cast.boolean === undefined || options.cast.boolean === null){ - // Cast boolean to string by default - options.cast.boolean = value => value ? '1' : ''; - } - // Normalize option cast.date - if(options.cast.date === undefined || options.cast.date === null){ - // Cast date to timestamp string by default - options.cast.date = value => '' + value.getTime(); - } - // Normalize option cast.number - if(options.cast.number === undefined || options.cast.number === null){ - // Cast number to string using native casting by default - options.cast.number = value => '' + value; - } - // Normalize option cast.object - if(options.cast.object === undefined || options.cast.object === null){ - // Stringify object as JSON by default - options.cast.object = value => JSON.stringify(value); - } - // Normalize option cast.string - if(options.cast.string === undefined || options.cast.string === null){ - // Leave string untouched - options.cast.string = function(value){return value;}; - } - // Normalize option `record_delimiter` - if(options.record_delimiter === undefined || options.record_delimiter === null){ - options.record_delimiter = '\n'; - }else if(isBuffer$1(options.record_delimiter)){ - options.record_delimiter = options.record_delimiter.toString(); - }else if(typeof options.record_delimiter !== 'string'){ - return Error(`Invalid Option: record_delimiter must be a buffer or a string, got ${JSON.stringify(options.record_delimiter)}`); - } - } - _transform(chunk, encoding, callback){ - if(this.state.stop === true){ - return; - } - const err = this.__transform(chunk); - if(err !== undefined){ - this.state.stop = true; - } - callback(err); + columns = newcolumns; } - _flush(callback){ - if(this.state.stop === true){ - // Note, Node.js 12 call flush even after an error, we must prevent - // `callback` from being called in flush without any error. - return; + return [undefined, columns]; + }; + + class CsvError extends Error { + constructor(code, message, ...contexts) { + if(Array.isArray(message)) message = message.join(' '); + super(message); + if(Error.captureStackTrace !== undefined){ + Error.captureStackTrace(this, CsvError); } - if(this.info.records === 0){ - this.bom(); - const err = this.headers(); - if(err) callback(err); + this.code = code; + for(const context of contexts){ + for(const key in context){ + const value = context[key]; + this[key] = isBuffer$1(value) ? value.toString() : value == null ? value : JSON.parse(JSON.stringify(value)); + } } - callback(); } - __transform(chunk){ - // Chunk validation - if(!Array.isArray(chunk) && typeof chunk !== 'object'){ - return Error(`Invalid Record: expect an array or an object, got ${JSON.stringify(chunk)}`); - } - // Detect columns from the first record - if(this.info.records === 0){ - if(Array.isArray(chunk)){ - if(this.options.header === true && this.options.columns === undefined){ - return Error('Undiscoverable Columns: header option requires column option or object records'); - } - }else if(this.options.columns === undefined){ - const [err, columns] = this.normalize_columns(Object.keys(chunk)); - if(err) return; - this.options.columns = columns; + } + + const underscore = function(str){ + return str.replace(/([A-Z])/g, function(_, match){ + return '_' + match.toLowerCase(); + }); + }; + + const normalize_options = function(opts) { + const options = {}; + // Merge with user options + for(const opt in opts){ + options[underscore(opt)] = opts[opt]; + } + // Normalize option `bom` + if(options.bom === undefined || options.bom === null || options.bom === false){ + options.bom = false; + }else if(options.bom !== true){ + return [new CsvError('CSV_OPTION_BOOLEAN_INVALID_TYPE', [ + 'option `bom` is optional and must be a boolean value,', + `got ${JSON.stringify(options.bom)}` + ])]; + } + // Normalize option `delimiter` + if(options.delimiter === undefined || options.delimiter === null){ + options.delimiter = ','; + }else if(isBuffer$1(options.delimiter)){ + options.delimiter = options.delimiter.toString(); + }else if(typeof options.delimiter !== 'string'){ + return [new CsvError('CSV_OPTION_DELIMITER_INVALID_TYPE', [ + 'option `delimiter` must be a buffer or a string,', + `got ${JSON.stringify(options.delimiter)}` + ])]; + } + // Normalize option `quote` + if(options.quote === undefined || options.quote === null){ + options.quote = '"'; + }else if(options.quote === true){ + options.quote = '"'; + }else if(options.quote === false){ + options.quote = ''; + }else if (isBuffer$1(options.quote)){ + options.quote = options.quote.toString(); + }else if(typeof options.quote !== 'string'){ + return [new CsvError('CSV_OPTION_QUOTE_INVALID_TYPE', [ + 'option `quote` must be a boolean, a buffer or a string,', + `got ${JSON.stringify(options.quote)}` + ])]; + } + // Normalize option `quoted` + if(options.quoted === undefined || options.quoted === null){ + options.quoted = false; + } + // Normalize option `quoted_empty` + if(options.quoted_empty === undefined || options.quoted_empty === null){ + options.quoted_empty = undefined; + } + // Normalize option `quoted_match` + if(options.quoted_match === undefined || options.quoted_match === null || options.quoted_match === false){ + options.quoted_match = null; + }else if(!Array.isArray(options.quoted_match)){ + options.quoted_match = [options.quoted_match]; + } + if(options.quoted_match){ + for(const quoted_match of options.quoted_match){ + const isString = typeof quoted_match === 'string'; + const isRegExp = quoted_match instanceof RegExp; + if(!isString && !isRegExp){ + return [Error(`Invalid Option: quoted_match must be a string or a regex, got ${JSON.stringify(quoted_match)}`)]; } } - // Emit the header - if(this.info.records === 0){ - this.bom(); - const err = this.headers(); - if(err) return err; - } - // Emit and stringify the record if an object or an array - try{ - this.emit('record', chunk, this.info.records); - }catch(err){ - return err; - } - // Convert the record into a string - let err, chunk_string; - if(this.options.eof){ - [err, chunk_string] = this.stringify(chunk); - if(err) return err; - if(chunk_string === undefined){ - return; - }else { - chunk_string = chunk_string + this.options.record_delimiter; + } + // Normalize option `quoted_string` + if(options.quoted_string === undefined || options.quoted_string === null){ + options.quoted_string = false; + } + // Normalize option `eof` + if(options.eof === undefined || options.eof === null){ + options.eof = true; + } + // Normalize option `escape` + if(options.escape === undefined || options.escape === null){ + options.escape = '"'; + }else if(isBuffer$1(options.escape)){ + options.escape = options.escape.toString(); + }else if(typeof options.escape !== 'string'){ + return [Error(`Invalid Option: escape must be a buffer or a string, got ${JSON.stringify(options.escape)}`)]; + } + if (options.escape.length > 1){ + return [Error(`Invalid Option: escape must be one character, got ${options.escape.length} characters`)]; + } + // Normalize option `header` + if(options.header === undefined || options.header === null){ + options.header = false; + } + // Normalize option `columns` + const [errColumns, columns] = normalize_columns(options.columns); + if(errColumns !== undefined) return [errColumns]; + options.columns = columns; + // Normalize option `quoted` + if(options.quoted === undefined || options.quoted === null){ + options.quoted = false; + } + // Normalize option `cast` + if(options.cast === undefined || options.cast === null){ + options.cast = {}; + } + // Normalize option cast.bigint + if(options.cast.bigint === undefined || options.cast.bigint === null){ + // Cast boolean to string by default + options.cast.bigint = value => '' + value; + } + // Normalize option cast.boolean + if(options.cast.boolean === undefined || options.cast.boolean === null){ + // Cast boolean to string by default + options.cast.boolean = value => value ? '1' : ''; + } + // Normalize option cast.date + if(options.cast.date === undefined || options.cast.date === null){ + // Cast date to timestamp string by default + options.cast.date = value => '' + value.getTime(); + } + // Normalize option cast.number + if(options.cast.number === undefined || options.cast.number === null){ + // Cast number to string using native casting by default + options.cast.number = value => '' + value; + } + // Normalize option cast.object + if(options.cast.object === undefined || options.cast.object === null){ + // Stringify object as JSON by default + options.cast.object = value => JSON.stringify(value); + } + // Normalize option cast.string + if(options.cast.string === undefined || options.cast.string === null){ + // Leave string untouched + options.cast.string = function(value){return value;}; + } + // Normalize option `on_record` + if(options.on_record !== undefined && typeof options.on_record !== 'function'){ + return [Error(`Invalid Option: "on_record" must be a function.`)]; + } + // Normalize option `record_delimiter` + if(options.record_delimiter === undefined || options.record_delimiter === null){ + options.record_delimiter = '\n'; + }else if(isBuffer$1(options.record_delimiter)){ + options.record_delimiter = options.record_delimiter.toString(); + }else if(typeof options.record_delimiter !== 'string'){ + return [Error(`Invalid Option: record_delimiter must be a buffer or a string, got ${JSON.stringify(options.record_delimiter)}`)]; + } + switch(options.record_delimiter){ + case 'auto': + options.record_delimiter = null; + break; + case 'unix': + options.record_delimiter = "\n"; + break; + case 'mac': + options.record_delimiter = "\r"; + break; + case 'windows': + options.record_delimiter = "\r\n"; + break; + case 'ascii': + options.record_delimiter = "\u001e"; + break; + case 'unicode': + options.record_delimiter = "\u2028"; + break; + } + return [undefined, options]; + }; + + const bom_utf8 = Buffer.from([239, 187, 191]); + + const stringifier = function(options, state, info){ + return { + options: options, + state: state, + info: info, + __transform: function(chunk, push){ + // Chunk validation + if(!Array.isArray(chunk) && typeof chunk !== 'object'){ + return Error(`Invalid Record: expect an array or an object, got ${JSON.stringify(chunk)}`); } - }else { - [err, chunk_string] = this.stringify(chunk); - if(err) return err; - if(chunk_string === undefined){ - return; - }else { - if(this.options.header || this.info.records){ - chunk_string = this.options.record_delimiter + chunk_string; + // Detect columns from the first record + if(this.info.records === 0){ + if(Array.isArray(chunk)){ + if(this.options.header === true && this.options.columns === undefined){ + return Error('Undiscoverable Columns: header option requires column option or object records'); + } + }else if(this.options.columns === undefined){ + const [err, columns] = normalize_columns(Object.keys(chunk)); + if(err) return; + this.options.columns = columns; } } - } - // Emit the csv - this.info.records++; - this.push(chunk_string); - } - stringify(chunk, chunkIsHeader=false){ - if(typeof chunk !== 'object'){ - return [undefined, chunk]; - } - const {columns} = this.options; - const record = []; - // Record is an array - if(Array.isArray(chunk)){ - // We are getting an array but the user has specified output columns. In - // this case, we respect the columns indexes - if(columns){ - chunk.splice(columns.length); + // Emit the header + if(this.info.records === 0){ + this.bom(push); + const err = this.headers(push); + if(err) return err; } - // Cast record elements - for(let i=0; i= 0; - const containsQuote = (quote !== '') && value.indexOf(quote) >= 0; - const containsEscape = value.indexOf(escape) >= 0 && (escape !== quote); - const containsRecordDelimiter = value.indexOf(record_delimiter) >= 0; - const quotedString = quoted_string && typeof field === 'string'; - let quotedMatch = quoted_match && quoted_match.filter(quoted_match => { - if(typeof quoted_match === 'string'){ - return value.indexOf(quoted_match) !== -1; - }else { - return quoted_match.test(value); + } + let csvrecord = ''; + for(let i=0; i 0; - const shouldQuote = containsQuote === true || containsdelimiter || containsRecordDelimiter || quoted || quotedString || quotedMatch; - if(shouldQuote === true && containsEscape === true){ - const regexp = escape === '\\' - ? new RegExp(escape + escape, 'g') - : new RegExp(escape, 'g'); - value = value.replace(regexp, escape + escape); + options = {...this.options, ...options}; + [err, options] = normalize_options(options); + if(err !== undefined){ + return [err]; + } + }else if(value === undefined || value === null){ + options = this.options; + }else { + return [Error(`Invalid Casting Value: returned value must return a string, an object, null or undefined, got ${JSON.stringify(value)}`)]; } - if(containsQuote === true){ - const regexp = new RegExp(quote,'g'); - value = value.replace(regexp, escape + quote); + const {delimiter, escape, quote, quoted, quoted_empty, quoted_string, quoted_match, record_delimiter} = options; + if(value){ + if(typeof value !== 'string'){ + return [Error(`Formatter must return a string, null or undefined, got ${JSON.stringify(value)}`)]; + } + const containsdelimiter = delimiter.length && value.indexOf(delimiter) >= 0; + const containsQuote = (quote !== '') && value.indexOf(quote) >= 0; + const containsEscape = value.indexOf(escape) >= 0 && (escape !== quote); + const containsRecordDelimiter = value.indexOf(record_delimiter) >= 0; + const quotedString = quoted_string && typeof field === 'string'; + let quotedMatch = quoted_match && quoted_match.filter(quoted_match => { + if(typeof quoted_match === 'string'){ + return value.indexOf(quoted_match) !== -1; + }else { + return quoted_match.test(value); + } + }); + quotedMatch = quotedMatch && quotedMatch.length > 0; + const shouldQuote = containsQuote === true || containsdelimiter || containsRecordDelimiter || quoted || quotedString || quotedMatch; + if(shouldQuote === true && containsEscape === true){ + const regexp = escape === '\\' + ? new RegExp(escape + escape, 'g') + : new RegExp(escape, 'g'); + value = value.replace(regexp, escape + escape); + } + if(containsQuote === true){ + const regexp = new RegExp(quote,'g'); + value = value.replace(regexp, escape + quote); + } + if(shouldQuote === true){ + value = quote + value + quote; + } + csvrecord += value; + }else if(quoted_empty === true || (field === '' && quoted_string === true && quoted_empty !== false)){ + csvrecord += quote + quote; } - if(shouldQuote === true){ - value = quote + value + quote; + if(i !== record.length - 1){ + csvrecord += delimiter; } - csvrecord += value; - }else if(quoted_empty === true || (field === '' && quoted_string === true && quoted_empty !== false)){ - csvrecord += quote + quote; } - if(i !== record.length - 1){ - csvrecord += delimiter; + return [undefined, csvrecord]; + }, + bom: function(push){ + if(this.options.bom !== true){ + return; } - } - return [undefined, csvrecord]; - } - bom(){ - if(this.options.bom !== true){ - return; - } - this.push(bom_utf8); - } - headers(){ - if(this.options.header === false){ - return; - } - if(this.options.columns === undefined){ - return; - } - let err; - let headers = this.options.columns.map(column => column.header); - if(this.options.eof){ - [err, headers] = this.stringify(headers, true); - headers += this.options.record_delimiter; - }else { - [err, headers] = this.stringify(headers); - } - if(err) return err; - this.push(headers); - } - __cast(value, context){ - const type = typeof value; - try{ - if(type === 'string'){ // Fine for 99% of the cases - return [undefined, this.options.cast.string(value, context)]; - }else if(type === 'bigint'){ - return [undefined, this.options.cast.bigint(value, context)]; - }else if(type === 'number'){ - return [undefined, this.options.cast.number(value, context)]; - }else if(type === 'boolean'){ - return [undefined, this.options.cast.boolean(value, context)]; - }else if(value instanceof Date){ - return [undefined, this.options.cast.date(value, context)]; - }else if(type === 'object' && value !== null){ - return [undefined, this.options.cast.object(value, context)]; - }else { - return [undefined, value, value]; + push(bom_utf8); + }, + headers: function(push){ + if(this.options.header === false){ + return; } - }catch(err){ - return [err]; - } - } - normalize_columns(columns){ - if(columns === undefined || columns === null){ - return []; - } - if(typeof columns !== 'object'){ - return [Error('Invalid option "columns": expect an array or an object')]; - } - if(!Array.isArray(columns)){ - const newcolumns = []; - for(const k in columns){ - newcolumns.push({ - key: k, - header: columns[k] - }); + if(this.options.columns === undefined){ + return; } - columns = newcolumns; - }else { - const newcolumns = []; - for(const column of columns){ - if(typeof column === 'string'){ - newcolumns.push({ - key: column, - header: column - }); - }else if(typeof column === 'object' && column !== undefined && !Array.isArray(column)){ - if(!column.key){ - return [Error('Invalid column definition: property "key" is required')]; - } - if(column.header === undefined){ - column.header = column.key; - } - newcolumns.push(column); + let err; + let headers = this.options.columns.map(column => column.header); + if(this.options.eof){ + [err, headers] = this.stringify(headers, true); + headers += this.options.record_delimiter; + }else { + [err, headers] = this.stringify(headers); + } + if(err) return err; + push(headers); + }, + __cast: function(value, context){ + const type = typeof value; + try{ + if(type === 'string'){ // Fine for 99% of the cases + return [undefined, this.options.cast.string(value, context)]; + }else if(type === 'bigint'){ + return [undefined, this.options.cast.bigint(value, context)]; + }else if(type === 'number'){ + return [undefined, this.options.cast.number(value, context)]; + }else if(type === 'boolean'){ + return [undefined, this.options.cast.boolean(value, context)]; + }else if(value instanceof Date){ + return [undefined, this.options.cast.date(value, context)]; + }else if(type === 'object' && value !== null){ + return [undefined, this.options.cast.object(value, context)]; }else { - return [Error('Invalid column definition: expect a string or an object')]; + return [undefined, value, value]; } + }catch(err){ + return [err]; } - columns = newcolumns; } - return [undefined, columns]; - } - } + }; + }; - const stringify = function(records, options={}){ + const stringify = function(records, opts={}){ const data = []; - const stringifier = new Stringifier(options); - stringifier.push = function(record){ - if(record === null){ - return; - } - data.push(record.toString()); + const [err, options] = normalize_options(opts); + if(err !== undefined) throw err; + const state = { + stop: false + }; + // Information + const info = { + records: 0 }; + const api = stringifier(options, state, info); + // stringifier.push = function(record){ + // if(record === null){ + // return; + // } + // data.push(record.toString()); + // }; for(const record of records){ - const err = stringifier.__transform(record, null); + const err = api.__transform(record, function(record){ + data.push(record); + }); if(err !== undefined) throw err; } return data.join(''); diff --git a/packages/stream-transform/test/option.parallel.coffee b/packages/stream-transform/test/option.parallel.coffee index 982af553a..55e8c7cb8 100644 --- a/packages/stream-transform/test/option.parallel.coffee +++ b/packages/stream-transform/test/option.parallel.coffee @@ -20,8 +20,8 @@ describe 'option.parallel', -> headers = -1 generator = generate length: 1000, objectMode: true, highWaterMark: 40, headers: 2, seed: 1, columns: [ (g) -> - letters pad 3, g._.count_created, '0' - (g) -> pad 3, g._.count_created-1, '0' + letters pad 3, g.state.count_created, '0' + (g) -> pad 3, g.state.count_created-1, '0' ] transformer = generator.pipe transform (record, next) -> count++ From f0315423ba576551f2bd08f3e1c3bc85e9003926 Mon Sep 17 00:00:00 2001 From: David Worms Date: Wed, 18 May 2022 16:54:10 +0200 Subject: [PATCH 11/19] fix(csv-generate): catch invalid value error --- packages/csv-generate/dist/cjs/index.cjs | 11 +++++++---- packages/csv-generate/dist/cjs/sync.cjs | 11 +++++++---- packages/csv-generate/dist/esm/index.js | 11 +++++++---- packages/csv-generate/dist/esm/sync.js | 11 +++++++---- packages/csv-generate/dist/iife/index.js | 11 +++++++---- packages/csv-generate/dist/iife/sync.js | 11 +++++++---- packages/csv-generate/dist/umd/index.js | 11 +++++++---- packages/csv-generate/dist/umd/sync.js | 11 +++++++---- packages/csv-generate/lib/api/read.js | 6 +++--- packages/csv-generate/lib/index.js | 5 ++++- packages/csv-generate/package.json | 2 +- 11 files changed, 64 insertions(+), 37 deletions(-) diff --git a/packages/csv-generate/dist/cjs/index.cjs b/packages/csv-generate/dist/cjs/index.cjs index ea7dcb621..61b8d6842 100644 --- a/packages/csv-generate/dist/cjs/index.cjs +++ b/packages/csv-generate/dist/cjs/index.cjs @@ -138,11 +138,11 @@ const read = (options, state, size, push, close) => { // Create the record let record = []; let recordLength; - options.columns.forEach((fn) => { + for(const fn of options.columns){ const result = fn({options: options, state: state}); const type = typeof result; if(result !== null && type !== 'string' && type !== 'number'){ - throw Error([ + return Error([ 'INVALID_VALUE:', 'values returned by column function must be', 'a string, a number or null,', @@ -150,7 +150,7 @@ const read = (options, state, size, push, close) => { ].join(' ')); } record.push(result); - }); + } // Obtain record length if(options.objectMode){ recordLength = 0; @@ -203,11 +203,14 @@ Generator.prototype.end = function(){ // Put new data into the read queue. Generator.prototype._read = function(size){ const self = this; - read(this.options, this.state, size, function(chunk) { + const err = read(this.options, this.state, size, function(chunk) { self.__push(chunk); }, function(){ self.push(null); }); + if(err){ + this.destroy(err); + } }; // Put new data into the read queue. Generator.prototype.__push = function(record){ diff --git a/packages/csv-generate/dist/cjs/sync.cjs b/packages/csv-generate/dist/cjs/sync.cjs index 2e3431ae0..7d08307f0 100644 --- a/packages/csv-generate/dist/cjs/sync.cjs +++ b/packages/csv-generate/dist/cjs/sync.cjs @@ -138,11 +138,11 @@ const read = (options, state, size, push, close) => { // Create the record let record = []; let recordLength; - options.columns.forEach((fn) => { + for(const fn of options.columns){ const result = fn({options: options, state: state}); const type = typeof result; if(result !== null && type !== 'string' && type !== 'number'){ - throw Error([ + return Error([ 'INVALID_VALUE:', 'values returned by column function must be', 'a string, a number or null,', @@ -150,7 +150,7 @@ const read = (options, state, size, push, close) => { ].join(' ')); } record.push(result); - }); + } // Obtain record length if(options.objectMode){ recordLength = 0; @@ -203,11 +203,14 @@ Generator.prototype.end = function(){ // Put new data into the read queue. Generator.prototype._read = function(size){ const self = this; - read(this.options, this.state, size, function(chunk) { + const err = read(this.options, this.state, size, function(chunk) { self.__push(chunk); }, function(){ self.push(null); }); + if(err){ + this.destroy(err); + } }; // Put new data into the read queue. Generator.prototype.__push = function(record){ diff --git a/packages/csv-generate/dist/esm/index.js b/packages/csv-generate/dist/esm/index.js index ebb52cc26..2bc51dc08 100644 --- a/packages/csv-generate/dist/esm/index.js +++ b/packages/csv-generate/dist/esm/index.js @@ -5164,11 +5164,11 @@ const read = (options, state, size, push, close) => { // Create the record let record = []; let recordLength; - options.columns.forEach((fn) => { + for(const fn of options.columns){ const result = fn({options: options, state: state}); const type = typeof result; if(result !== null && type !== 'string' && type !== 'number'){ - throw Error([ + return Error([ 'INVALID_VALUE:', 'values returned by column function must be', 'a string, a number or null,', @@ -5176,7 +5176,7 @@ const read = (options, state, size, push, close) => { ].join(' ')); } record.push(result); - }); + } // Obtain record length if(options.objectMode){ recordLength = 0; @@ -5229,11 +5229,14 @@ Generator.prototype.end = function(){ // Put new data into the read queue. Generator.prototype._read = function(size){ const self = this; - read(this.options, this.state, size, function(chunk) { + const err = read(this.options, this.state, size, function(chunk) { self.__push(chunk); }, function(){ self.push(null); }); + if(err){ + this.destroy(err); + } }; // Put new data into the read queue. Generator.prototype.__push = function(record){ diff --git a/packages/csv-generate/dist/esm/sync.js b/packages/csv-generate/dist/esm/sync.js index 576c7f81c..fcfdee9a5 100644 --- a/packages/csv-generate/dist/esm/sync.js +++ b/packages/csv-generate/dist/esm/sync.js @@ -5164,11 +5164,11 @@ const read = (options, state, size, push, close) => { // Create the record let record = []; let recordLength; - options.columns.forEach((fn) => { + for(const fn of options.columns){ const result = fn({options: options, state: state}); const type = typeof result; if(result !== null && type !== 'string' && type !== 'number'){ - throw Error([ + return Error([ 'INVALID_VALUE:', 'values returned by column function must be', 'a string, a number or null,', @@ -5176,7 +5176,7 @@ const read = (options, state, size, push, close) => { ].join(' ')); } record.push(result); - }); + } // Obtain record length if(options.objectMode){ recordLength = 0; @@ -5229,11 +5229,14 @@ Generator.prototype.end = function(){ // Put new data into the read queue. Generator.prototype._read = function(size){ const self = this; - read(this.options, this.state, size, function(chunk) { + const err = read(this.options, this.state, size, function(chunk) { self.__push(chunk); }, function(){ self.push(null); }); + if(err){ + this.destroy(err); + } }; // Put new data into the read queue. Generator.prototype.__push = function(record){ diff --git a/packages/csv-generate/dist/iife/index.js b/packages/csv-generate/dist/iife/index.js index 65360d158..cff7f9712 100644 --- a/packages/csv-generate/dist/iife/index.js +++ b/packages/csv-generate/dist/iife/index.js @@ -5167,11 +5167,11 @@ var csv_generate = (function (exports) { // Create the record let record = []; let recordLength; - options.columns.forEach((fn) => { + for(const fn of options.columns){ const result = fn({options: options, state: state}); const type = typeof result; if(result !== null && type !== 'string' && type !== 'number'){ - throw Error([ + return Error([ 'INVALID_VALUE:', 'values returned by column function must be', 'a string, a number or null,', @@ -5179,7 +5179,7 @@ var csv_generate = (function (exports) { ].join(' ')); } record.push(result); - }); + } // Obtain record length if(options.objectMode){ recordLength = 0; @@ -5232,11 +5232,14 @@ var csv_generate = (function (exports) { // Put new data into the read queue. Generator.prototype._read = function(size){ const self = this; - read(this.options, this.state, size, function(chunk) { + const err = read(this.options, this.state, size, function(chunk) { self.__push(chunk); }, function(){ self.push(null); }); + if(err){ + this.destroy(err); + } }; // Put new data into the read queue. Generator.prototype.__push = function(record){ diff --git a/packages/csv-generate/dist/iife/sync.js b/packages/csv-generate/dist/iife/sync.js index addb15a09..d263fef64 100644 --- a/packages/csv-generate/dist/iife/sync.js +++ b/packages/csv-generate/dist/iife/sync.js @@ -5167,11 +5167,11 @@ var csv_generate_sync = (function (exports) { // Create the record let record = []; let recordLength; - options.columns.forEach((fn) => { + for(const fn of options.columns){ const result = fn({options: options, state: state}); const type = typeof result; if(result !== null && type !== 'string' && type !== 'number'){ - throw Error([ + return Error([ 'INVALID_VALUE:', 'values returned by column function must be', 'a string, a number or null,', @@ -5179,7 +5179,7 @@ var csv_generate_sync = (function (exports) { ].join(' ')); } record.push(result); - }); + } // Obtain record length if(options.objectMode){ recordLength = 0; @@ -5232,11 +5232,14 @@ var csv_generate_sync = (function (exports) { // Put new data into the read queue. Generator.prototype._read = function(size){ const self = this; - read(this.options, this.state, size, function(chunk) { + const err = read(this.options, this.state, size, function(chunk) { self.__push(chunk); }, function(){ self.push(null); }); + if(err){ + this.destroy(err); + } }; // Put new data into the read queue. Generator.prototype.__push = function(record){ diff --git a/packages/csv-generate/dist/umd/index.js b/packages/csv-generate/dist/umd/index.js index 86ff176ef..29643e818 100644 --- a/packages/csv-generate/dist/umd/index.js +++ b/packages/csv-generate/dist/umd/index.js @@ -5170,11 +5170,11 @@ // Create the record let record = []; let recordLength; - options.columns.forEach((fn) => { + for(const fn of options.columns){ const result = fn({options: options, state: state}); const type = typeof result; if(result !== null && type !== 'string' && type !== 'number'){ - throw Error([ + return Error([ 'INVALID_VALUE:', 'values returned by column function must be', 'a string, a number or null,', @@ -5182,7 +5182,7 @@ ].join(' ')); } record.push(result); - }); + } // Obtain record length if(options.objectMode){ recordLength = 0; @@ -5235,11 +5235,14 @@ // Put new data into the read queue. Generator.prototype._read = function(size){ const self = this; - read(this.options, this.state, size, function(chunk) { + const err = read(this.options, this.state, size, function(chunk) { self.__push(chunk); }, function(){ self.push(null); }); + if(err){ + this.destroy(err); + } }; // Put new data into the read queue. Generator.prototype.__push = function(record){ diff --git a/packages/csv-generate/dist/umd/sync.js b/packages/csv-generate/dist/umd/sync.js index 109cece5e..274094f3d 100644 --- a/packages/csv-generate/dist/umd/sync.js +++ b/packages/csv-generate/dist/umd/sync.js @@ -5170,11 +5170,11 @@ // Create the record let record = []; let recordLength; - options.columns.forEach((fn) => { + for(const fn of options.columns){ const result = fn({options: options, state: state}); const type = typeof result; if(result !== null && type !== 'string' && type !== 'number'){ - throw Error([ + return Error([ 'INVALID_VALUE:', 'values returned by column function must be', 'a string, a number or null,', @@ -5182,7 +5182,7 @@ ].join(' ')); } record.push(result); - }); + } // Obtain record length if(options.objectMode){ recordLength = 0; @@ -5235,11 +5235,14 @@ // Put new data into the read queue. Generator.prototype._read = function(size){ const self = this; - read(this.options, this.state, size, function(chunk) { + const err = read(this.options, this.state, size, function(chunk) { self.__push(chunk); }, function(){ self.push(null); }); + if(err){ + this.destroy(err); + } }; // Put new data into the read queue. Generator.prototype.__push = function(record){ diff --git a/packages/csv-generate/lib/api/read.js b/packages/csv-generate/lib/api/read.js index e4191a3bb..9cca0412c 100644 --- a/packages/csv-generate/lib/api/read.js +++ b/packages/csv-generate/lib/api/read.js @@ -29,11 +29,11 @@ const read = (options, state, size, push, close) => { // Create the record let record = []; let recordLength; - options.columns.forEach((fn) => { + for(const fn of options.columns){ const result = fn({options: options, state: state}); const type = typeof result; if(result !== null && type !== 'string' && type !== 'number'){ - throw Error([ + return Error([ 'INVALID_VALUE:', 'values returned by column function must be', 'a string, a number or null,', @@ -41,7 +41,7 @@ const read = (options, state, size, push, close) => { ].join(' ')); } record.push(result); - }); + } // Obtain record length if(options.objectMode){ recordLength = 0; diff --git a/packages/csv-generate/lib/index.js b/packages/csv-generate/lib/index.js index 2ab0b4d3b..b066bc45f 100644 --- a/packages/csv-generate/lib/index.js +++ b/packages/csv-generate/lib/index.js @@ -26,11 +26,14 @@ Generator.prototype.end = function(){ // Put new data into the read queue. Generator.prototype._read = function(size){ const self = this; - read(this.options, this.state, size, function(chunk) { + const err = read(this.options, this.state, size, function(chunk) { self.__push(chunk); }, function(){ self.push(null); }); + if(err){ + this.destroy(err); + } }; // Put new data into the read queue. Generator.prototype.__push = function(record){ diff --git a/packages/csv-generate/package.json b/packages/csv-generate/package.json index 5ea6fbfb7..709cc17a5 100644 --- a/packages/csv-generate/package.json +++ b/packages/csv-generate/package.json @@ -77,7 +77,7 @@ "preversion": "npm run build && git add dist", "pretest": "npm run build", "test": "mocha 'test/**/*.{coffee,ts}'", - "test:legacy": "mocha --loader=./test/loaders/legacy/all.js 'test/**/*.{coffee,ts}'" + "test:legacy": "mocha --loader=./test/loaders/legacy/all.js 'test/**/*.{coffee,ts}' --ignore test/api.web_stream.coffee" }, "type": "module", "types": "dist/esm/index.d.ts", From 23f626080e99d56dabee3605c5e51ace90de46ba Mon Sep 17 00:00:00 2001 From: David Worms Date: Thu, 19 May 2022 00:57:47 +0200 Subject: [PATCH 12/19] test: fix legacy node support in parse and stringify --- packages/csv-parse/package.json | 2 +- packages/csv-stringify/package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/csv-parse/package.json b/packages/csv-parse/package.json index 3074ec110..0af278837 100644 --- a/packages/csv-parse/package.json +++ b/packages/csv-parse/package.json @@ -96,7 +96,7 @@ "preversion": "npm run build && git add dist", "pretest": "npm run build", "test": "mocha 'test/**/*.{coffee,ts}'", - "test:legacy": "mocha --loader=./test/loaders/legacy/all.js 'test/**/*.{coffee,ts}'" + "test:legacy": "mocha --loader=./test/loaders/legacy/all.js 'test/**/*.{coffee,ts}' --ignore test/api.web_stream.coffee" }, "type": "module", "types": "dist/esm/index.d.ts", diff --git a/packages/csv-stringify/package.json b/packages/csv-stringify/package.json index 88370f16b..ffd9070c4 100644 --- a/packages/csv-stringify/package.json +++ b/packages/csv-stringify/package.json @@ -73,7 +73,7 @@ "preversion": "npm run build && git add dist", "pretest": "npm run build", "test": "mocha 'test/**/*.{coffee,ts}'", - "test:legacy": "mocha --loader=./test/loaders/legacy/all.js 'test/**/*.{coffee,ts}'" + "test:legacy": "mocha --loader=./test/loaders/legacy/all.js 'test/**/*.{coffee,ts}' --ignore test/api.web_stream.coffee" }, "type": "module", "types": "dist/esm/index.d.ts", From 60efa7862ed43bd2fd19d1f027a1809e9df6a67e Mon Sep 17 00:00:00 2001 From: Sam Magura Date: Tue, 24 May 2022 02:57:01 -0400 Subject: [PATCH 13/19] fix(csv-stringify): update TS types to allow cast to return an object (#339) --- packages/csv-stringify/lib/index.d.ts | 19 ++++++++++++++++++- packages/csv-stringify/test/api.types.sync.ts | 12 ++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/packages/csv-stringify/lib/index.d.ts b/packages/csv-stringify/lib/index.d.ts index a1cfbd525..ccfafefc2 100644 --- a/packages/csv-stringify/lib/index.d.ts +++ b/packages/csv-stringify/lib/index.d.ts @@ -4,7 +4,24 @@ import * as stream from "stream"; export type Callback = (err: Error | undefined, output: string) => void export type RecordDelimiter = string | Buffer | 'auto' | 'unix' | 'mac' | 'windows' | 'ascii' | 'unicode' -export type Cast = (value: T, context: CastingContext) => string + +export type CastReturnObject = { value: string } & Pick< + Options, + | 'delimiter' + | 'escape' + | 'quote' + | 'quoted' + | 'quoted_empty' + | 'quoted_string' + | 'quoted_match' + | 'record_delimiter' +> + +export type Cast = ( + value: T, + context: CastingContext +) => string | CastReturnObject; + export type PlainObject = Record export type Input = any[] export interface ColumnOption { diff --git a/packages/csv-stringify/test/api.types.sync.ts b/packages/csv-stringify/test/api.types.sync.ts index 5d1593213..e2ba3761c 100644 --- a/packages/csv-stringify/test/api.types.sync.ts +++ b/packages/csv-stringify/test/api.types.sync.ts @@ -37,5 +37,17 @@ describe('API Types', () => { } return options }) + + it('allows cast to return an object', () => { + const options: Options = { + cast: { + boolean: (value: boolean) => ({ + value: value.toString(), + delimiter: ';', + quote: false + }) + } + } + }) }) From c719ddc165e702e980c8ba4122db9590d9b2e504 Mon Sep 17 00:00:00 2001 From: David Worms Date: Tue, 24 May 2022 09:16:38 +0200 Subject: [PATCH 14/19] chore: latest dependencies --- demo/browser/package.json | 2 +- demo/cjs/package.json | 10 +- demo/eslint/package.json | 4 +- demo/esm/package.json | 4 +- demo/issues-cjs/package.json | 4 +- demo/issues-esm/package.json | 4 +- demo/webpack-ts/package.json | 6 +- demo/webpack/package.json | 2 +- packages/csv-generate/dist/esm/index.js | 100 ++++++++++++++++- packages/csv-generate/dist/esm/sync.js | 100 ++++++++++++++++- packages/csv-generate/dist/iife/index.js | 100 ++++++++++++++++- packages/csv-generate/dist/iife/sync.js | 100 ++++++++++++++++- packages/csv-generate/dist/umd/index.js | 100 ++++++++++++++++- packages/csv-generate/dist/umd/sync.js | 100 ++++++++++++++++- packages/csv-generate/package.json | 20 ++-- packages/csv-parse/dist/esm/index.js | 100 ++++++++++++++++- packages/csv-parse/dist/esm/sync.js | 7 ++ packages/csv-parse/dist/iife/index.js | 100 ++++++++++++++++- packages/csv-parse/dist/iife/sync.js | 7 ++ packages/csv-parse/dist/umd/index.js | 100 ++++++++++++++++- packages/csv-parse/dist/umd/sync.js | 7 ++ packages/csv-parse/package.json | 20 ++-- packages/csv-stringify/dist/cjs/index.d.ts | 19 +++- packages/csv-stringify/dist/esm/index.d.ts | 19 +++- packages/csv-stringify/dist/esm/index.js | 100 ++++++++++++++++- packages/csv-stringify/dist/esm/sync.js | 7 ++ packages/csv-stringify/dist/iife/index.js | 100 ++++++++++++++++- packages/csv-stringify/dist/iife/sync.js | 7 ++ packages/csv-stringify/dist/umd/index.js | 100 ++++++++++++++++- packages/csv-stringify/dist/umd/sync.js | 7 ++ packages/csv-stringify/package.json | 22 ++-- packages/csv/dist/cjs/index.cjs | 11 +- packages/csv/dist/cjs/sync.cjs | 11 +- packages/csv/dist/esm/index.js | 111 +++++++++++++++++-- packages/csv/dist/esm/sync.js | 111 +++++++++++++++++-- packages/csv/dist/iife/index.js | 111 +++++++++++++++++-- packages/csv/dist/iife/sync.js | 111 +++++++++++++++++-- packages/csv/dist/umd/index.js | 111 +++++++++++++++++-- packages/csv/dist/umd/sync.js | 111 +++++++++++++++++-- packages/csv/package.json | 20 ++-- packages/stream-transform/dist/esm/index.js | 100 ++++++++++++++++- packages/stream-transform/dist/esm/sync.js | 100 ++++++++++++++++- packages/stream-transform/dist/iife/index.js | 100 ++++++++++++++++- packages/stream-transform/dist/iife/sync.js | 100 ++++++++++++++++- packages/stream-transform/dist/umd/index.js | 100 ++++++++++++++++- packages/stream-transform/dist/umd/sync.js | 100 ++++++++++++++++- packages/stream-transform/package.json | 20 ++-- 47 files changed, 2531 insertions(+), 175 deletions(-) diff --git a/demo/browser/package.json b/demo/browser/package.json index 9bbfa66cb..68bdcc54d 100644 --- a/demo/browser/package.json +++ b/demo/browser/package.json @@ -20,6 +20,6 @@ "start": "node server.js" }, "dependencies": { - "express": "^4.17.2" + "express": "^4.18.1" } } diff --git a/demo/cjs/package.json b/demo/cjs/package.json index b7900b45d..50bf40dea 100644 --- a/demo/cjs/package.json +++ b/demo/cjs/package.json @@ -6,12 +6,12 @@ "type": "commonjs", "private": true, "devDependencies": { - "@types/node": "^17.0.18", - "coffeescript": "^2.6.1", - "mocha": "^9.2.0", + "@types/node": "^17.0.35", + "coffeescript": "^2.7.0", + "mocha": "^10.0.0", "should": "^13.2.3", - "ts-node": "^10.5.0", - "typescript": "^4.5.5" + "ts-node": "^10.8.0", + "typescript": "^4.6.4" }, "mocha": { "inline-diffs": true, diff --git a/demo/eslint/package.json b/demo/eslint/package.json index f273cab22..ad5f58aeb 100644 --- a/demo/eslint/package.json +++ b/demo/eslint/package.json @@ -12,8 +12,8 @@ "csv-stringify": "^6.0.5" }, "devDependencies": { - "eslint": "^8.11.0", + "eslint": "^8.16.0", "eslint-config-airbnb-base": "^15.0.0", - "eslint-plugin-import": "^2.25.4" + "eslint-plugin-import": "^2.26.0" } } diff --git a/demo/esm/package.json b/demo/esm/package.json index a74891283..1a44334ae 100644 --- a/demo/esm/package.json +++ b/demo/esm/package.json @@ -10,8 +10,8 @@ "csv-parse": "^5.0.4" }, "devDependencies": { - "coffeescript": "^2.6.1", - "mocha": "^9.2.0", + "coffeescript": "^2.7.0", + "mocha": "^10.0.0", "should": "^13.2.3" }, "mocha": { diff --git a/demo/issues-cjs/package.json b/demo/issues-cjs/package.json index 707f0e1e8..777750ef0 100644 --- a/demo/issues-cjs/package.json +++ b/demo/issues-cjs/package.json @@ -5,8 +5,8 @@ "license": "MIT", "private": true, "devDependencies": { - "coffeescript": "^2.6.1", - "mocha": "^9.2.0", + "coffeescript": "^2.7.0", + "mocha": "^10.0.0", "should": "^13.2.3" }, "mocha": { diff --git a/demo/issues-esm/package.json b/demo/issues-esm/package.json index ed90dd0de..7f667d9ce 100644 --- a/demo/issues-esm/package.json +++ b/demo/issues-esm/package.json @@ -6,9 +6,9 @@ "type": "module", "private": true, "devDependencies": { - "coffeescript": "^2.6.1", + "coffeescript": "^2.7.0", "dirname-filename-esm": "^1.1.1", - "mocha": "^9.2.0", + "mocha": "^10.0.0", "should": "^13.2.3" }, "mocha": { diff --git a/demo/webpack-ts/package.json b/demo/webpack-ts/package.json index ee4d7649a..182e1172b 100644 --- a/demo/webpack-ts/package.json +++ b/demo/webpack-ts/package.json @@ -15,9 +15,9 @@ "buffer-browserify": "^0.2.5", "node-polyfill-webpack-plugin": "^1.1.4", "stream-browserify": "^3.0.0", - "ts-loader": "^9.2.6", - "typescript": "^4.5.5", - "webpack": "^5.69.0", + "ts-loader": "^9.3.0", + "typescript": "^4.6.4", + "webpack": "^5.72.1", "webpack-cli": "^4.9.2" }, "dependencies": { diff --git a/demo/webpack/package.json b/demo/webpack/package.json index 2864d1d08..106b7052b 100644 --- a/demo/webpack/package.json +++ b/demo/webpack/package.json @@ -12,7 +12,7 @@ "author": "", "license": "ISC", "devDependencies": { - "webpack": "^5.69.0", + "webpack": "^5.72.1", "webpack-cli": "^4.9.2" }, "dependencies": { diff --git a/packages/csv-generate/dist/esm/index.js b/packages/csv-generate/dist/esm/index.js index 2bc51dc08..cdd66c3c2 100644 --- a/packages/csv-generate/dist/esm/index.js +++ b/packages/csv-generate/dist/esm/index.js @@ -233,6 +233,11 @@ Buffer.TYPED_ARRAY_SUPPORT = global$1.TYPED_ARRAY_SUPPORT !== undefined ? global$1.TYPED_ARRAY_SUPPORT : true; +/* + * Export kMaxLength after typed array support is determined. + */ +kMaxLength(); + function kMaxLength () { return Buffer.TYPED_ARRAY_SUPPORT ? 0x7fffffff @@ -324,6 +329,8 @@ Buffer.from = function (value, encodingOrOffset, length) { if (Buffer.TYPED_ARRAY_SUPPORT) { Buffer.prototype.__proto__ = Uint8Array.prototype; Buffer.__proto__ = Uint8Array; + if (typeof Symbol !== 'undefined' && Symbol.species && + Buffer[Symbol.species] === Buffer) ; } function assertSize (size) { @@ -2562,16 +2569,95 @@ function Item(fun, array) { Item.prototype.run = function () { this.fun.apply(null, this.array); }; +var title = 'browser'; +var platform = 'browser'; +var browser = true; +var env = {}; +var argv = []; +var version = ''; // empty string to avoid regexp issues +var versions = {}; +var release = {}; +var config = {}; + +function noop() {} + +var on = noop; +var addListener = noop; +var once = noop; +var off = noop; +var removeListener = noop; +var removeAllListeners = noop; +var emit = noop; + +function binding(name) { + throw new Error('process.binding is not supported'); +} + +function cwd () { return '/' } +function chdir (dir) { + throw new Error('process.chdir is not supported'); +}function umask() { return 0; } // from https://github.com/kumavis/browser-process-hrtime/blob/master/index.js var performance = global$1.performance || {}; -performance.now || +var performanceNow = + performance.now || performance.mozNow || performance.msNow || performance.oNow || performance.webkitNow || function(){ return (new Date()).getTime() }; +// generate timestamp or delta +// see http://nodejs.org/api/process.html#process_process_hrtime +function hrtime(previousTimestamp){ + var clocktime = performanceNow.call(performance)*1e-3; + var seconds = Math.floor(clocktime); + var nanoseconds = Math.floor((clocktime%1)*1e9); + if (previousTimestamp) { + seconds = seconds - previousTimestamp[0]; + nanoseconds = nanoseconds - previousTimestamp[1]; + if (nanoseconds<0) { + seconds--; + nanoseconds += 1e9; + } + } + return [seconds,nanoseconds] +} + +var startTime = new Date(); +function uptime() { + var currentTime = new Date(); + var dif = currentTime - startTime; + return dif / 1000; +} + +var process = { + nextTick: nextTick, + title: title, + browser: browser, + env: env, + argv: argv, + version: version, + versions: versions, + on: on, + addListener: addListener, + once: once, + off: off, + removeListener: removeListener, + removeAllListeners: removeAllListeners, + emit: emit, + binding: binding, + cwd: cwd, + chdir: chdir, + umask: umask, + hrtime: hrtime, + platform: platform, + release: release, + config: config, + uptime: uptime +}; + var inherits; if (typeof Object.create === 'function'){ inherits = function inherits(ctor, superCtor) { @@ -2647,10 +2733,18 @@ function deprecate(fn, msg) { }; } + if (process.noDeprecation === true) { + return fn; + } + var warned = false; function deprecated() { if (!warned) { - { + if (process.throwDeprecation) { + throw new Error(msg); + } else if (process.traceDeprecation) { + console.trace(msg); + } else { console.error(msg); } warned = true; @@ -2665,7 +2759,7 @@ var debugs = {}; var debugEnviron; function debuglog(set) { if (isUndefined(debugEnviron)) - debugEnviron = ''; + debugEnviron = process.env.NODE_DEBUG || ''; set = set.toUpperCase(); if (!debugs[set]) { if (new RegExp('\\b' + set + '\\b', 'i').test(debugEnviron)) { diff --git a/packages/csv-generate/dist/esm/sync.js b/packages/csv-generate/dist/esm/sync.js index fcfdee9a5..f67f66b7c 100644 --- a/packages/csv-generate/dist/esm/sync.js +++ b/packages/csv-generate/dist/esm/sync.js @@ -233,6 +233,11 @@ Buffer.TYPED_ARRAY_SUPPORT = global$1.TYPED_ARRAY_SUPPORT !== undefined ? global$1.TYPED_ARRAY_SUPPORT : true; +/* + * Export kMaxLength after typed array support is determined. + */ +kMaxLength(); + function kMaxLength () { return Buffer.TYPED_ARRAY_SUPPORT ? 0x7fffffff @@ -324,6 +329,8 @@ Buffer.from = function (value, encodingOrOffset, length) { if (Buffer.TYPED_ARRAY_SUPPORT) { Buffer.prototype.__proto__ = Uint8Array.prototype; Buffer.__proto__ = Uint8Array; + if (typeof Symbol !== 'undefined' && Symbol.species && + Buffer[Symbol.species] === Buffer) ; } function assertSize (size) { @@ -2562,16 +2569,95 @@ function Item(fun, array) { Item.prototype.run = function () { this.fun.apply(null, this.array); }; +var title = 'browser'; +var platform = 'browser'; +var browser = true; +var env = {}; +var argv = []; +var version = ''; // empty string to avoid regexp issues +var versions = {}; +var release = {}; +var config = {}; + +function noop() {} + +var on = noop; +var addListener = noop; +var once = noop; +var off = noop; +var removeListener = noop; +var removeAllListeners = noop; +var emit = noop; + +function binding(name) { + throw new Error('process.binding is not supported'); +} + +function cwd () { return '/' } +function chdir (dir) { + throw new Error('process.chdir is not supported'); +}function umask() { return 0; } // from https://github.com/kumavis/browser-process-hrtime/blob/master/index.js var performance = global$1.performance || {}; -performance.now || +var performanceNow = + performance.now || performance.mozNow || performance.msNow || performance.oNow || performance.webkitNow || function(){ return (new Date()).getTime() }; +// generate timestamp or delta +// see http://nodejs.org/api/process.html#process_process_hrtime +function hrtime(previousTimestamp){ + var clocktime = performanceNow.call(performance)*1e-3; + var seconds = Math.floor(clocktime); + var nanoseconds = Math.floor((clocktime%1)*1e9); + if (previousTimestamp) { + seconds = seconds - previousTimestamp[0]; + nanoseconds = nanoseconds - previousTimestamp[1]; + if (nanoseconds<0) { + seconds--; + nanoseconds += 1e9; + } + } + return [seconds,nanoseconds] +} + +var startTime = new Date(); +function uptime() { + var currentTime = new Date(); + var dif = currentTime - startTime; + return dif / 1000; +} + +var process = { + nextTick: nextTick, + title: title, + browser: browser, + env: env, + argv: argv, + version: version, + versions: versions, + on: on, + addListener: addListener, + once: once, + off: off, + removeListener: removeListener, + removeAllListeners: removeAllListeners, + emit: emit, + binding: binding, + cwd: cwd, + chdir: chdir, + umask: umask, + hrtime: hrtime, + platform: platform, + release: release, + config: config, + uptime: uptime +}; + var inherits; if (typeof Object.create === 'function'){ inherits = function inherits(ctor, superCtor) { @@ -2647,10 +2733,18 @@ function deprecate(fn, msg) { }; } + if (process.noDeprecation === true) { + return fn; + } + var warned = false; function deprecated() { if (!warned) { - { + if (process.throwDeprecation) { + throw new Error(msg); + } else if (process.traceDeprecation) { + console.trace(msg); + } else { console.error(msg); } warned = true; @@ -2665,7 +2759,7 @@ var debugs = {}; var debugEnviron; function debuglog(set) { if (isUndefined(debugEnviron)) - debugEnviron = ''; + debugEnviron = process.env.NODE_DEBUG || ''; set = set.toUpperCase(); if (!debugs[set]) { if (new RegExp('\\b' + set + '\\b', 'i').test(debugEnviron)) { diff --git a/packages/csv-generate/dist/iife/index.js b/packages/csv-generate/dist/iife/index.js index cff7f9712..d5234852f 100644 --- a/packages/csv-generate/dist/iife/index.js +++ b/packages/csv-generate/dist/iife/index.js @@ -236,6 +236,11 @@ var csv_generate = (function (exports) { ? global$1.TYPED_ARRAY_SUPPORT : true; + /* + * Export kMaxLength after typed array support is determined. + */ + kMaxLength(); + function kMaxLength () { return Buffer.TYPED_ARRAY_SUPPORT ? 0x7fffffff @@ -327,6 +332,8 @@ var csv_generate = (function (exports) { if (Buffer.TYPED_ARRAY_SUPPORT) { Buffer.prototype.__proto__ = Uint8Array.prototype; Buffer.__proto__ = Uint8Array; + if (typeof Symbol !== 'undefined' && Symbol.species && + Buffer[Symbol.species] === Buffer) ; } function assertSize (size) { @@ -2565,16 +2572,95 @@ var csv_generate = (function (exports) { Item.prototype.run = function () { this.fun.apply(null, this.array); }; + var title = 'browser'; + var platform = 'browser'; + var browser = true; + var env = {}; + var argv = []; + var version = ''; // empty string to avoid regexp issues + var versions = {}; + var release = {}; + var config = {}; + + function noop() {} + + var on = noop; + var addListener = noop; + var once = noop; + var off = noop; + var removeListener = noop; + var removeAllListeners = noop; + var emit = noop; + + function binding(name) { + throw new Error('process.binding is not supported'); + } + + function cwd () { return '/' } + function chdir (dir) { + throw new Error('process.chdir is not supported'); + }function umask() { return 0; } // from https://github.com/kumavis/browser-process-hrtime/blob/master/index.js var performance = global$1.performance || {}; - performance.now || + var performanceNow = + performance.now || performance.mozNow || performance.msNow || performance.oNow || performance.webkitNow || function(){ return (new Date()).getTime() }; + // generate timestamp or delta + // see http://nodejs.org/api/process.html#process_process_hrtime + function hrtime(previousTimestamp){ + var clocktime = performanceNow.call(performance)*1e-3; + var seconds = Math.floor(clocktime); + var nanoseconds = Math.floor((clocktime%1)*1e9); + if (previousTimestamp) { + seconds = seconds - previousTimestamp[0]; + nanoseconds = nanoseconds - previousTimestamp[1]; + if (nanoseconds<0) { + seconds--; + nanoseconds += 1e9; + } + } + return [seconds,nanoseconds] + } + + var startTime = new Date(); + function uptime() { + var currentTime = new Date(); + var dif = currentTime - startTime; + return dif / 1000; + } + + var process = { + nextTick: nextTick, + title: title, + browser: browser, + env: env, + argv: argv, + version: version, + versions: versions, + on: on, + addListener: addListener, + once: once, + off: off, + removeListener: removeListener, + removeAllListeners: removeAllListeners, + emit: emit, + binding: binding, + cwd: cwd, + chdir: chdir, + umask: umask, + hrtime: hrtime, + platform: platform, + release: release, + config: config, + uptime: uptime + }; + var inherits; if (typeof Object.create === 'function'){ inherits = function inherits(ctor, superCtor) { @@ -2650,10 +2736,18 @@ var csv_generate = (function (exports) { }; } + if (process.noDeprecation === true) { + return fn; + } + var warned = false; function deprecated() { if (!warned) { - { + if (process.throwDeprecation) { + throw new Error(msg); + } else if (process.traceDeprecation) { + console.trace(msg); + } else { console.error(msg); } warned = true; @@ -2668,7 +2762,7 @@ var csv_generate = (function (exports) { var debugEnviron; function debuglog(set) { if (isUndefined(debugEnviron)) - debugEnviron = ''; + debugEnviron = process.env.NODE_DEBUG || ''; set = set.toUpperCase(); if (!debugs[set]) { if (new RegExp('\\b' + set + '\\b', 'i').test(debugEnviron)) { diff --git a/packages/csv-generate/dist/iife/sync.js b/packages/csv-generate/dist/iife/sync.js index d263fef64..c20ba6cf2 100644 --- a/packages/csv-generate/dist/iife/sync.js +++ b/packages/csv-generate/dist/iife/sync.js @@ -236,6 +236,11 @@ var csv_generate_sync = (function (exports) { ? global$1.TYPED_ARRAY_SUPPORT : true; + /* + * Export kMaxLength after typed array support is determined. + */ + kMaxLength(); + function kMaxLength () { return Buffer.TYPED_ARRAY_SUPPORT ? 0x7fffffff @@ -327,6 +332,8 @@ var csv_generate_sync = (function (exports) { if (Buffer.TYPED_ARRAY_SUPPORT) { Buffer.prototype.__proto__ = Uint8Array.prototype; Buffer.__proto__ = Uint8Array; + if (typeof Symbol !== 'undefined' && Symbol.species && + Buffer[Symbol.species] === Buffer) ; } function assertSize (size) { @@ -2565,16 +2572,95 @@ var csv_generate_sync = (function (exports) { Item.prototype.run = function () { this.fun.apply(null, this.array); }; + var title = 'browser'; + var platform = 'browser'; + var browser = true; + var env = {}; + var argv = []; + var version = ''; // empty string to avoid regexp issues + var versions = {}; + var release = {}; + var config = {}; + + function noop() {} + + var on = noop; + var addListener = noop; + var once = noop; + var off = noop; + var removeListener = noop; + var removeAllListeners = noop; + var emit = noop; + + function binding(name) { + throw new Error('process.binding is not supported'); + } + + function cwd () { return '/' } + function chdir (dir) { + throw new Error('process.chdir is not supported'); + }function umask() { return 0; } // from https://github.com/kumavis/browser-process-hrtime/blob/master/index.js var performance = global$1.performance || {}; - performance.now || + var performanceNow = + performance.now || performance.mozNow || performance.msNow || performance.oNow || performance.webkitNow || function(){ return (new Date()).getTime() }; + // generate timestamp or delta + // see http://nodejs.org/api/process.html#process_process_hrtime + function hrtime(previousTimestamp){ + var clocktime = performanceNow.call(performance)*1e-3; + var seconds = Math.floor(clocktime); + var nanoseconds = Math.floor((clocktime%1)*1e9); + if (previousTimestamp) { + seconds = seconds - previousTimestamp[0]; + nanoseconds = nanoseconds - previousTimestamp[1]; + if (nanoseconds<0) { + seconds--; + nanoseconds += 1e9; + } + } + return [seconds,nanoseconds] + } + + var startTime = new Date(); + function uptime() { + var currentTime = new Date(); + var dif = currentTime - startTime; + return dif / 1000; + } + + var process = { + nextTick: nextTick, + title: title, + browser: browser, + env: env, + argv: argv, + version: version, + versions: versions, + on: on, + addListener: addListener, + once: once, + off: off, + removeListener: removeListener, + removeAllListeners: removeAllListeners, + emit: emit, + binding: binding, + cwd: cwd, + chdir: chdir, + umask: umask, + hrtime: hrtime, + platform: platform, + release: release, + config: config, + uptime: uptime + }; + var inherits; if (typeof Object.create === 'function'){ inherits = function inherits(ctor, superCtor) { @@ -2650,10 +2736,18 @@ var csv_generate_sync = (function (exports) { }; } + if (process.noDeprecation === true) { + return fn; + } + var warned = false; function deprecated() { if (!warned) { - { + if (process.throwDeprecation) { + throw new Error(msg); + } else if (process.traceDeprecation) { + console.trace(msg); + } else { console.error(msg); } warned = true; @@ -2668,7 +2762,7 @@ var csv_generate_sync = (function (exports) { var debugEnviron; function debuglog(set) { if (isUndefined(debugEnviron)) - debugEnviron = ''; + debugEnviron = process.env.NODE_DEBUG || ''; set = set.toUpperCase(); if (!debugs[set]) { if (new RegExp('\\b' + set + '\\b', 'i').test(debugEnviron)) { diff --git a/packages/csv-generate/dist/umd/index.js b/packages/csv-generate/dist/umd/index.js index 29643e818..71a18c525 100644 --- a/packages/csv-generate/dist/umd/index.js +++ b/packages/csv-generate/dist/umd/index.js @@ -239,6 +239,11 @@ ? global$1.TYPED_ARRAY_SUPPORT : true; + /* + * Export kMaxLength after typed array support is determined. + */ + kMaxLength(); + function kMaxLength () { return Buffer.TYPED_ARRAY_SUPPORT ? 0x7fffffff @@ -330,6 +335,8 @@ if (Buffer.TYPED_ARRAY_SUPPORT) { Buffer.prototype.__proto__ = Uint8Array.prototype; Buffer.__proto__ = Uint8Array; + if (typeof Symbol !== 'undefined' && Symbol.species && + Buffer[Symbol.species] === Buffer) ; } function assertSize (size) { @@ -2568,16 +2575,95 @@ Item.prototype.run = function () { this.fun.apply(null, this.array); }; + var title = 'browser'; + var platform = 'browser'; + var browser = true; + var env = {}; + var argv = []; + var version = ''; // empty string to avoid regexp issues + var versions = {}; + var release = {}; + var config = {}; + + function noop() {} + + var on = noop; + var addListener = noop; + var once = noop; + var off = noop; + var removeListener = noop; + var removeAllListeners = noop; + var emit = noop; + + function binding(name) { + throw new Error('process.binding is not supported'); + } + + function cwd () { return '/' } + function chdir (dir) { + throw new Error('process.chdir is not supported'); + }function umask() { return 0; } // from https://github.com/kumavis/browser-process-hrtime/blob/master/index.js var performance = global$1.performance || {}; - performance.now || + var performanceNow = + performance.now || performance.mozNow || performance.msNow || performance.oNow || performance.webkitNow || function(){ return (new Date()).getTime() }; + // generate timestamp or delta + // see http://nodejs.org/api/process.html#process_process_hrtime + function hrtime(previousTimestamp){ + var clocktime = performanceNow.call(performance)*1e-3; + var seconds = Math.floor(clocktime); + var nanoseconds = Math.floor((clocktime%1)*1e9); + if (previousTimestamp) { + seconds = seconds - previousTimestamp[0]; + nanoseconds = nanoseconds - previousTimestamp[1]; + if (nanoseconds<0) { + seconds--; + nanoseconds += 1e9; + } + } + return [seconds,nanoseconds] + } + + var startTime = new Date(); + function uptime() { + var currentTime = new Date(); + var dif = currentTime - startTime; + return dif / 1000; + } + + var process = { + nextTick: nextTick, + title: title, + browser: browser, + env: env, + argv: argv, + version: version, + versions: versions, + on: on, + addListener: addListener, + once: once, + off: off, + removeListener: removeListener, + removeAllListeners: removeAllListeners, + emit: emit, + binding: binding, + cwd: cwd, + chdir: chdir, + umask: umask, + hrtime: hrtime, + platform: platform, + release: release, + config: config, + uptime: uptime + }; + var inherits; if (typeof Object.create === 'function'){ inherits = function inherits(ctor, superCtor) { @@ -2653,10 +2739,18 @@ }; } + if (process.noDeprecation === true) { + return fn; + } + var warned = false; function deprecated() { if (!warned) { - { + if (process.throwDeprecation) { + throw new Error(msg); + } else if (process.traceDeprecation) { + console.trace(msg); + } else { console.error(msg); } warned = true; @@ -2671,7 +2765,7 @@ var debugEnviron; function debuglog(set) { if (isUndefined(debugEnviron)) - debugEnviron = ''; + debugEnviron = process.env.NODE_DEBUG || ''; set = set.toUpperCase(); if (!debugs[set]) { if (new RegExp('\\b' + set + '\\b', 'i').test(debugEnviron)) { diff --git a/packages/csv-generate/dist/umd/sync.js b/packages/csv-generate/dist/umd/sync.js index 274094f3d..6d66dc3de 100644 --- a/packages/csv-generate/dist/umd/sync.js +++ b/packages/csv-generate/dist/umd/sync.js @@ -239,6 +239,11 @@ ? global$1.TYPED_ARRAY_SUPPORT : true; + /* + * Export kMaxLength after typed array support is determined. + */ + kMaxLength(); + function kMaxLength () { return Buffer.TYPED_ARRAY_SUPPORT ? 0x7fffffff @@ -330,6 +335,8 @@ if (Buffer.TYPED_ARRAY_SUPPORT) { Buffer.prototype.__proto__ = Uint8Array.prototype; Buffer.__proto__ = Uint8Array; + if (typeof Symbol !== 'undefined' && Symbol.species && + Buffer[Symbol.species] === Buffer) ; } function assertSize (size) { @@ -2568,16 +2575,95 @@ Item.prototype.run = function () { this.fun.apply(null, this.array); }; + var title = 'browser'; + var platform = 'browser'; + var browser = true; + var env = {}; + var argv = []; + var version = ''; // empty string to avoid regexp issues + var versions = {}; + var release = {}; + var config = {}; + + function noop() {} + + var on = noop; + var addListener = noop; + var once = noop; + var off = noop; + var removeListener = noop; + var removeAllListeners = noop; + var emit = noop; + + function binding(name) { + throw new Error('process.binding is not supported'); + } + + function cwd () { return '/' } + function chdir (dir) { + throw new Error('process.chdir is not supported'); + }function umask() { return 0; } // from https://github.com/kumavis/browser-process-hrtime/blob/master/index.js var performance = global$1.performance || {}; - performance.now || + var performanceNow = + performance.now || performance.mozNow || performance.msNow || performance.oNow || performance.webkitNow || function(){ return (new Date()).getTime() }; + // generate timestamp or delta + // see http://nodejs.org/api/process.html#process_process_hrtime + function hrtime(previousTimestamp){ + var clocktime = performanceNow.call(performance)*1e-3; + var seconds = Math.floor(clocktime); + var nanoseconds = Math.floor((clocktime%1)*1e9); + if (previousTimestamp) { + seconds = seconds - previousTimestamp[0]; + nanoseconds = nanoseconds - previousTimestamp[1]; + if (nanoseconds<0) { + seconds--; + nanoseconds += 1e9; + } + } + return [seconds,nanoseconds] + } + + var startTime = new Date(); + function uptime() { + var currentTime = new Date(); + var dif = currentTime - startTime; + return dif / 1000; + } + + var process = { + nextTick: nextTick, + title: title, + browser: browser, + env: env, + argv: argv, + version: version, + versions: versions, + on: on, + addListener: addListener, + once: once, + off: off, + removeListener: removeListener, + removeAllListeners: removeAllListeners, + emit: emit, + binding: binding, + cwd: cwd, + chdir: chdir, + umask: umask, + hrtime: hrtime, + platform: platform, + release: release, + config: config, + uptime: uptime + }; + var inherits; if (typeof Object.create === 'function'){ inherits = function inherits(ctor, superCtor) { @@ -2653,10 +2739,18 @@ }; } + if (process.noDeprecation === true) { + return fn; + } + var warned = false; function deprecated() { if (!warned) { - { + if (process.throwDeprecation) { + throw new Error(msg); + } else if (process.traceDeprecation) { + console.trace(msg); + } else { console.error(msg); } warned = true; @@ -2671,7 +2765,7 @@ var debugEnviron; function debuglog(set) { if (isUndefined(debugEnviron)) - debugEnviron = ''; + debugEnviron = process.env.NODE_DEBUG || ''; set = set.toUpperCase(); if (!debugs[set]) { if (new RegExp('\\b' + set + '\\b', 'i').test(debugEnviron)) { diff --git a/packages/csv-generate/package.json b/packages/csv-generate/package.json index 709cc17a5..1f3d2acfa 100644 --- a/packages/csv-generate/package.json +++ b/packages/csv-generate/package.json @@ -11,21 +11,21 @@ "bugs": "https://github.com/adaltas/node-csv-generate/issues", "author": "David Worms (https://www.adaltas.com)", "devDependencies": { - "@rollup/plugin-eslint": "^8.0.1", - "@rollup/plugin-node-resolve": "^13.1.3", - "@types/mocha": "^9.1.0", - "@types/node": "^17.0.18", + "@rollup/plugin-eslint": "^8.0.2", + "@rollup/plugin-node-resolve": "^13.3.0", + "@types/mocha": "^9.1.1", + "@types/node": "^17.0.35", "@types/should": "^13.0.0", - "coffeescript": "~2.6.1", + "coffeescript": "~2.7.0", "each": "^1.2.2", - "eslint": "^8.9.0", - "mocha": "~9.2.0", - "rollup": "^2.67.2", + "eslint": "^8.16.0", + "mocha": "~10.0.0", + "rollup": "^2.74.1", "rollup-plugin-node-builtins": "^2.1.2", "rollup-plugin-node-globals": "^1.4.0", "should": "~13.2.3", - "ts-node": "^10.5.0", - "typescript": "^4.5.5" + "ts-node": "^10.8.0", + "typescript": "^4.6.4" }, "exports": { ".": { diff --git a/packages/csv-parse/dist/esm/index.js b/packages/csv-parse/dist/esm/index.js index 614a434ab..7d2d939c8 100644 --- a/packages/csv-parse/dist/esm/index.js +++ b/packages/csv-parse/dist/esm/index.js @@ -233,6 +233,11 @@ Buffer.TYPED_ARRAY_SUPPORT = global$1.TYPED_ARRAY_SUPPORT !== undefined ? global$1.TYPED_ARRAY_SUPPORT : true; +/* + * Export kMaxLength after typed array support is determined. + */ +kMaxLength(); + function kMaxLength () { return Buffer.TYPED_ARRAY_SUPPORT ? 0x7fffffff @@ -324,6 +329,8 @@ Buffer.from = function (value, encodingOrOffset, length) { if (Buffer.TYPED_ARRAY_SUPPORT) { Buffer.prototype.__proto__ = Uint8Array.prototype; Buffer.__proto__ = Uint8Array; + if (typeof Symbol !== 'undefined' && Symbol.species && + Buffer[Symbol.species] === Buffer) ; } function assertSize (size) { @@ -2562,16 +2569,95 @@ function Item(fun, array) { Item.prototype.run = function () { this.fun.apply(null, this.array); }; +var title = 'browser'; +var platform = 'browser'; +var browser = true; +var env = {}; +var argv = []; +var version = ''; // empty string to avoid regexp issues +var versions = {}; +var release = {}; +var config = {}; + +function noop() {} + +var on = noop; +var addListener = noop; +var once = noop; +var off = noop; +var removeListener = noop; +var removeAllListeners = noop; +var emit = noop; + +function binding(name) { + throw new Error('process.binding is not supported'); +} + +function cwd () { return '/' } +function chdir (dir) { + throw new Error('process.chdir is not supported'); +}function umask() { return 0; } // from https://github.com/kumavis/browser-process-hrtime/blob/master/index.js var performance = global$1.performance || {}; -performance.now || +var performanceNow = + performance.now || performance.mozNow || performance.msNow || performance.oNow || performance.webkitNow || function(){ return (new Date()).getTime() }; +// generate timestamp or delta +// see http://nodejs.org/api/process.html#process_process_hrtime +function hrtime(previousTimestamp){ + var clocktime = performanceNow.call(performance)*1e-3; + var seconds = Math.floor(clocktime); + var nanoseconds = Math.floor((clocktime%1)*1e9); + if (previousTimestamp) { + seconds = seconds - previousTimestamp[0]; + nanoseconds = nanoseconds - previousTimestamp[1]; + if (nanoseconds<0) { + seconds--; + nanoseconds += 1e9; + } + } + return [seconds,nanoseconds] +} + +var startTime = new Date(); +function uptime() { + var currentTime = new Date(); + var dif = currentTime - startTime; + return dif / 1000; +} + +var process = { + nextTick: nextTick, + title: title, + browser: browser, + env: env, + argv: argv, + version: version, + versions: versions, + on: on, + addListener: addListener, + once: once, + off: off, + removeListener: removeListener, + removeAllListeners: removeAllListeners, + emit: emit, + binding: binding, + cwd: cwd, + chdir: chdir, + umask: umask, + hrtime: hrtime, + platform: platform, + release: release, + config: config, + uptime: uptime +}; + var inherits; if (typeof Object.create === 'function'){ inherits = function inherits(ctor, superCtor) { @@ -2647,10 +2733,18 @@ function deprecate(fn, msg) { }; } + if (process.noDeprecation === true) { + return fn; + } + var warned = false; function deprecated() { if (!warned) { - { + if (process.throwDeprecation) { + throw new Error(msg); + } else if (process.traceDeprecation) { + console.trace(msg); + } else { console.error(msg); } warned = true; @@ -2665,7 +2759,7 @@ var debugs = {}; var debugEnviron; function debuglog(set) { if (isUndefined(debugEnviron)) - debugEnviron = ''; + debugEnviron = process.env.NODE_DEBUG || ''; set = set.toUpperCase(); if (!debugs[set]) { if (new RegExp('\\b' + set + '\\b', 'i').test(debugEnviron)) { diff --git a/packages/csv-parse/dist/esm/sync.js b/packages/csv-parse/dist/esm/sync.js index cdef53688..6330dca8d 100644 --- a/packages/csv-parse/dist/esm/sync.js +++ b/packages/csv-parse/dist/esm/sync.js @@ -233,6 +233,11 @@ Buffer.TYPED_ARRAY_SUPPORT = global$1.TYPED_ARRAY_SUPPORT !== undefined ? global$1.TYPED_ARRAY_SUPPORT : true; +/* + * Export kMaxLength after typed array support is determined. + */ +kMaxLength(); + function kMaxLength () { return Buffer.TYPED_ARRAY_SUPPORT ? 0x7fffffff @@ -324,6 +329,8 @@ Buffer.from = function (value, encodingOrOffset, length) { if (Buffer.TYPED_ARRAY_SUPPORT) { Buffer.prototype.__proto__ = Uint8Array.prototype; Buffer.__proto__ = Uint8Array; + if (typeof Symbol !== 'undefined' && Symbol.species && + Buffer[Symbol.species] === Buffer) ; } function assertSize (size) { diff --git a/packages/csv-parse/dist/iife/index.js b/packages/csv-parse/dist/iife/index.js index e0e38d690..ed4b1adbe 100644 --- a/packages/csv-parse/dist/iife/index.js +++ b/packages/csv-parse/dist/iife/index.js @@ -236,6 +236,11 @@ var csv_parse = (function (exports) { ? global$1.TYPED_ARRAY_SUPPORT : true; + /* + * Export kMaxLength after typed array support is determined. + */ + kMaxLength(); + function kMaxLength () { return Buffer.TYPED_ARRAY_SUPPORT ? 0x7fffffff @@ -327,6 +332,8 @@ var csv_parse = (function (exports) { if (Buffer.TYPED_ARRAY_SUPPORT) { Buffer.prototype.__proto__ = Uint8Array.prototype; Buffer.__proto__ = Uint8Array; + if (typeof Symbol !== 'undefined' && Symbol.species && + Buffer[Symbol.species] === Buffer) ; } function assertSize (size) { @@ -2565,16 +2572,95 @@ var csv_parse = (function (exports) { Item.prototype.run = function () { this.fun.apply(null, this.array); }; + var title = 'browser'; + var platform = 'browser'; + var browser = true; + var env = {}; + var argv = []; + var version = ''; // empty string to avoid regexp issues + var versions = {}; + var release = {}; + var config = {}; + + function noop() {} + + var on = noop; + var addListener = noop; + var once = noop; + var off = noop; + var removeListener = noop; + var removeAllListeners = noop; + var emit = noop; + + function binding(name) { + throw new Error('process.binding is not supported'); + } + + function cwd () { return '/' } + function chdir (dir) { + throw new Error('process.chdir is not supported'); + }function umask() { return 0; } // from https://github.com/kumavis/browser-process-hrtime/blob/master/index.js var performance = global$1.performance || {}; - performance.now || + var performanceNow = + performance.now || performance.mozNow || performance.msNow || performance.oNow || performance.webkitNow || function(){ return (new Date()).getTime() }; + // generate timestamp or delta + // see http://nodejs.org/api/process.html#process_process_hrtime + function hrtime(previousTimestamp){ + var clocktime = performanceNow.call(performance)*1e-3; + var seconds = Math.floor(clocktime); + var nanoseconds = Math.floor((clocktime%1)*1e9); + if (previousTimestamp) { + seconds = seconds - previousTimestamp[0]; + nanoseconds = nanoseconds - previousTimestamp[1]; + if (nanoseconds<0) { + seconds--; + nanoseconds += 1e9; + } + } + return [seconds,nanoseconds] + } + + var startTime = new Date(); + function uptime() { + var currentTime = new Date(); + var dif = currentTime - startTime; + return dif / 1000; + } + + var process = { + nextTick: nextTick, + title: title, + browser: browser, + env: env, + argv: argv, + version: version, + versions: versions, + on: on, + addListener: addListener, + once: once, + off: off, + removeListener: removeListener, + removeAllListeners: removeAllListeners, + emit: emit, + binding: binding, + cwd: cwd, + chdir: chdir, + umask: umask, + hrtime: hrtime, + platform: platform, + release: release, + config: config, + uptime: uptime + }; + var inherits; if (typeof Object.create === 'function'){ inherits = function inherits(ctor, superCtor) { @@ -2650,10 +2736,18 @@ var csv_parse = (function (exports) { }; } + if (process.noDeprecation === true) { + return fn; + } + var warned = false; function deprecated() { if (!warned) { - { + if (process.throwDeprecation) { + throw new Error(msg); + } else if (process.traceDeprecation) { + console.trace(msg); + } else { console.error(msg); } warned = true; @@ -2668,7 +2762,7 @@ var csv_parse = (function (exports) { var debugEnviron; function debuglog(set) { if (isUndefined(debugEnviron)) - debugEnviron = ''; + debugEnviron = process.env.NODE_DEBUG || ''; set = set.toUpperCase(); if (!debugs[set]) { if (new RegExp('\\b' + set + '\\b', 'i').test(debugEnviron)) { diff --git a/packages/csv-parse/dist/iife/sync.js b/packages/csv-parse/dist/iife/sync.js index 8aefdf38e..1985b2214 100644 --- a/packages/csv-parse/dist/iife/sync.js +++ b/packages/csv-parse/dist/iife/sync.js @@ -236,6 +236,11 @@ var csv_parse_sync = (function (exports) { ? global$1.TYPED_ARRAY_SUPPORT : true; + /* + * Export kMaxLength after typed array support is determined. + */ + kMaxLength(); + function kMaxLength () { return Buffer.TYPED_ARRAY_SUPPORT ? 0x7fffffff @@ -327,6 +332,8 @@ var csv_parse_sync = (function (exports) { if (Buffer.TYPED_ARRAY_SUPPORT) { Buffer.prototype.__proto__ = Uint8Array.prototype; Buffer.__proto__ = Uint8Array; + if (typeof Symbol !== 'undefined' && Symbol.species && + Buffer[Symbol.species] === Buffer) ; } function assertSize (size) { diff --git a/packages/csv-parse/dist/umd/index.js b/packages/csv-parse/dist/umd/index.js index 5223def8f..f8c415ad6 100644 --- a/packages/csv-parse/dist/umd/index.js +++ b/packages/csv-parse/dist/umd/index.js @@ -239,6 +239,11 @@ ? global$1.TYPED_ARRAY_SUPPORT : true; + /* + * Export kMaxLength after typed array support is determined. + */ + kMaxLength(); + function kMaxLength () { return Buffer.TYPED_ARRAY_SUPPORT ? 0x7fffffff @@ -330,6 +335,8 @@ if (Buffer.TYPED_ARRAY_SUPPORT) { Buffer.prototype.__proto__ = Uint8Array.prototype; Buffer.__proto__ = Uint8Array; + if (typeof Symbol !== 'undefined' && Symbol.species && + Buffer[Symbol.species] === Buffer) ; } function assertSize (size) { @@ -2568,16 +2575,95 @@ Item.prototype.run = function () { this.fun.apply(null, this.array); }; + var title = 'browser'; + var platform = 'browser'; + var browser = true; + var env = {}; + var argv = []; + var version = ''; // empty string to avoid regexp issues + var versions = {}; + var release = {}; + var config = {}; + + function noop() {} + + var on = noop; + var addListener = noop; + var once = noop; + var off = noop; + var removeListener = noop; + var removeAllListeners = noop; + var emit = noop; + + function binding(name) { + throw new Error('process.binding is not supported'); + } + + function cwd () { return '/' } + function chdir (dir) { + throw new Error('process.chdir is not supported'); + }function umask() { return 0; } // from https://github.com/kumavis/browser-process-hrtime/blob/master/index.js var performance = global$1.performance || {}; - performance.now || + var performanceNow = + performance.now || performance.mozNow || performance.msNow || performance.oNow || performance.webkitNow || function(){ return (new Date()).getTime() }; + // generate timestamp or delta + // see http://nodejs.org/api/process.html#process_process_hrtime + function hrtime(previousTimestamp){ + var clocktime = performanceNow.call(performance)*1e-3; + var seconds = Math.floor(clocktime); + var nanoseconds = Math.floor((clocktime%1)*1e9); + if (previousTimestamp) { + seconds = seconds - previousTimestamp[0]; + nanoseconds = nanoseconds - previousTimestamp[1]; + if (nanoseconds<0) { + seconds--; + nanoseconds += 1e9; + } + } + return [seconds,nanoseconds] + } + + var startTime = new Date(); + function uptime() { + var currentTime = new Date(); + var dif = currentTime - startTime; + return dif / 1000; + } + + var process = { + nextTick: nextTick, + title: title, + browser: browser, + env: env, + argv: argv, + version: version, + versions: versions, + on: on, + addListener: addListener, + once: once, + off: off, + removeListener: removeListener, + removeAllListeners: removeAllListeners, + emit: emit, + binding: binding, + cwd: cwd, + chdir: chdir, + umask: umask, + hrtime: hrtime, + platform: platform, + release: release, + config: config, + uptime: uptime + }; + var inherits; if (typeof Object.create === 'function'){ inherits = function inherits(ctor, superCtor) { @@ -2653,10 +2739,18 @@ }; } + if (process.noDeprecation === true) { + return fn; + } + var warned = false; function deprecated() { if (!warned) { - { + if (process.throwDeprecation) { + throw new Error(msg); + } else if (process.traceDeprecation) { + console.trace(msg); + } else { console.error(msg); } warned = true; @@ -2671,7 +2765,7 @@ var debugEnviron; function debuglog(set) { if (isUndefined(debugEnviron)) - debugEnviron = ''; + debugEnviron = process.env.NODE_DEBUG || ''; set = set.toUpperCase(); if (!debugs[set]) { if (new RegExp('\\b' + set + '\\b', 'i').test(debugEnviron)) { diff --git a/packages/csv-parse/dist/umd/sync.js b/packages/csv-parse/dist/umd/sync.js index 28133103c..b896d9fe2 100644 --- a/packages/csv-parse/dist/umd/sync.js +++ b/packages/csv-parse/dist/umd/sync.js @@ -239,6 +239,11 @@ ? global$1.TYPED_ARRAY_SUPPORT : true; + /* + * Export kMaxLength after typed array support is determined. + */ + kMaxLength(); + function kMaxLength () { return Buffer.TYPED_ARRAY_SUPPORT ? 0x7fffffff @@ -330,6 +335,8 @@ if (Buffer.TYPED_ARRAY_SUPPORT) { Buffer.prototype.__proto__ = Uint8Array.prototype; Buffer.__proto__ = Uint8Array; + if (typeof Symbol !== 'undefined' && Symbol.species && + Buffer[Symbol.species] === Buffer) ; } function assertSize (size) { diff --git a/packages/csv-parse/package.json b/packages/csv-parse/package.json index 0af278837..15e0994cd 100644 --- a/packages/csv-parse/package.json +++ b/packages/csv-parse/package.json @@ -42,25 +42,25 @@ "./browser/esm/sync": "./dist/esm/sync.js" }, "devDependencies": { - "@rollup/plugin-eslint": "^8.0.1", - "@rollup/plugin-node-resolve": "^13.1.3", - "@types/mocha": "^9.1.0", - "@types/node": "^17.0.18", + "@rollup/plugin-eslint": "^8.0.2", + "@rollup/plugin-node-resolve": "^13.3.0", + "@types/mocha": "^9.1.1", + "@types/node": "^17.0.35", "coffeelint": "^2.1.0", - "coffeescript": "^2.6.1", + "coffeescript": "^2.7.0", "csv-generate": "^4.0.4", "csv-spectrum": "^1.0.0", "each": "^1.2.2", - "eslint": "^8.9.0", - "mocha": "^9.2.0", + "eslint": "^8.16.0", + "mocha": "^10.0.0", "pad": "^3.2.0", - "rollup": "^2.67.2", + "rollup": "^2.74.1", "rollup-plugin-node-builtins": "^2.1.2", "rollup-plugin-node-globals": "^1.4.0", "should": "^13.2.3", "stream-transform": "^3.0.4", - "ts-node": "^10.5.0", - "typescript": "^4.5.5" + "ts-node": "^10.8.0", + "typescript": "^4.6.4" }, "files": [ "dist", diff --git a/packages/csv-stringify/dist/cjs/index.d.ts b/packages/csv-stringify/dist/cjs/index.d.ts index a1cfbd525..ccfafefc2 100644 --- a/packages/csv-stringify/dist/cjs/index.d.ts +++ b/packages/csv-stringify/dist/cjs/index.d.ts @@ -4,7 +4,24 @@ import * as stream from "stream"; export type Callback = (err: Error | undefined, output: string) => void export type RecordDelimiter = string | Buffer | 'auto' | 'unix' | 'mac' | 'windows' | 'ascii' | 'unicode' -export type Cast = (value: T, context: CastingContext) => string + +export type CastReturnObject = { value: string } & Pick< + Options, + | 'delimiter' + | 'escape' + | 'quote' + | 'quoted' + | 'quoted_empty' + | 'quoted_string' + | 'quoted_match' + | 'record_delimiter' +> + +export type Cast = ( + value: T, + context: CastingContext +) => string | CastReturnObject; + export type PlainObject = Record export type Input = any[] export interface ColumnOption { diff --git a/packages/csv-stringify/dist/esm/index.d.ts b/packages/csv-stringify/dist/esm/index.d.ts index a1cfbd525..ccfafefc2 100644 --- a/packages/csv-stringify/dist/esm/index.d.ts +++ b/packages/csv-stringify/dist/esm/index.d.ts @@ -4,7 +4,24 @@ import * as stream from "stream"; export type Callback = (err: Error | undefined, output: string) => void export type RecordDelimiter = string | Buffer | 'auto' | 'unix' | 'mac' | 'windows' | 'ascii' | 'unicode' -export type Cast = (value: T, context: CastingContext) => string + +export type CastReturnObject = { value: string } & Pick< + Options, + | 'delimiter' + | 'escape' + | 'quote' + | 'quoted' + | 'quoted_empty' + | 'quoted_string' + | 'quoted_match' + | 'record_delimiter' +> + +export type Cast = ( + value: T, + context: CastingContext +) => string | CastReturnObject; + export type PlainObject = Record export type Input = any[] export interface ColumnOption { diff --git a/packages/csv-stringify/dist/esm/index.js b/packages/csv-stringify/dist/esm/index.js index 7992dafef..6263661d5 100644 --- a/packages/csv-stringify/dist/esm/index.js +++ b/packages/csv-stringify/dist/esm/index.js @@ -699,6 +699,11 @@ Buffer.TYPED_ARRAY_SUPPORT = global$1.TYPED_ARRAY_SUPPORT !== undefined ? global$1.TYPED_ARRAY_SUPPORT : true; +/* + * Export kMaxLength after typed array support is determined. + */ +kMaxLength(); + function kMaxLength () { return Buffer.TYPED_ARRAY_SUPPORT ? 0x7fffffff @@ -790,6 +795,8 @@ Buffer.from = function (value, encodingOrOffset, length) { if (Buffer.TYPED_ARRAY_SUPPORT) { Buffer.prototype.__proto__ = Uint8Array.prototype; Buffer.__proto__ = Uint8Array; + if (typeof Symbol !== 'undefined' && Symbol.species && + Buffer[Symbol.species] === Buffer) ; } function assertSize (size) { @@ -2562,16 +2569,95 @@ function Item(fun, array) { Item.prototype.run = function () { this.fun.apply(null, this.array); }; +var title = 'browser'; +var platform = 'browser'; +var browser = true; +var env = {}; +var argv = []; +var version = ''; // empty string to avoid regexp issues +var versions = {}; +var release = {}; +var config = {}; + +function noop() {} + +var on = noop; +var addListener = noop; +var once = noop; +var off = noop; +var removeListener = noop; +var removeAllListeners = noop; +var emit = noop; + +function binding(name) { + throw new Error('process.binding is not supported'); +} + +function cwd () { return '/' } +function chdir (dir) { + throw new Error('process.chdir is not supported'); +}function umask() { return 0; } // from https://github.com/kumavis/browser-process-hrtime/blob/master/index.js var performance = global$1.performance || {}; -performance.now || +var performanceNow = + performance.now || performance.mozNow || performance.msNow || performance.oNow || performance.webkitNow || function(){ return (new Date()).getTime() }; +// generate timestamp or delta +// see http://nodejs.org/api/process.html#process_process_hrtime +function hrtime(previousTimestamp){ + var clocktime = performanceNow.call(performance)*1e-3; + var seconds = Math.floor(clocktime); + var nanoseconds = Math.floor((clocktime%1)*1e9); + if (previousTimestamp) { + seconds = seconds - previousTimestamp[0]; + nanoseconds = nanoseconds - previousTimestamp[1]; + if (nanoseconds<0) { + seconds--; + nanoseconds += 1e9; + } + } + return [seconds,nanoseconds] +} + +var startTime = new Date(); +function uptime() { + var currentTime = new Date(); + var dif = currentTime - startTime; + return dif / 1000; +} + +var process = { + nextTick: nextTick, + title: title, + browser: browser, + env: env, + argv: argv, + version: version, + versions: versions, + on: on, + addListener: addListener, + once: once, + off: off, + removeListener: removeListener, + removeAllListeners: removeAllListeners, + emit: emit, + binding: binding, + cwd: cwd, + chdir: chdir, + umask: umask, + hrtime: hrtime, + platform: platform, + release: release, + config: config, + uptime: uptime +}; + var inherits; if (typeof Object.create === 'function'){ inherits = function inherits(ctor, superCtor) { @@ -2647,10 +2733,18 @@ function deprecate(fn, msg) { }; } + if (process.noDeprecation === true) { + return fn; + } + var warned = false; function deprecated() { if (!warned) { - { + if (process.throwDeprecation) { + throw new Error(msg); + } else if (process.traceDeprecation) { + console.trace(msg); + } else { console.error(msg); } warned = true; @@ -2665,7 +2759,7 @@ var debugs = {}; var debugEnviron; function debuglog(set) { if (isUndefined(debugEnviron)) - debugEnviron = ''; + debugEnviron = process.env.NODE_DEBUG || ''; set = set.toUpperCase(); if (!debugs[set]) { if (new RegExp('\\b' + set + '\\b', 'i').test(debugEnviron)) { diff --git a/packages/csv-stringify/dist/esm/sync.js b/packages/csv-stringify/dist/esm/sync.js index 9a9b37d4d..579cc0b24 100644 --- a/packages/csv-stringify/dist/esm/sync.js +++ b/packages/csv-stringify/dist/esm/sync.js @@ -233,6 +233,11 @@ Buffer.TYPED_ARRAY_SUPPORT = global$1.TYPED_ARRAY_SUPPORT !== undefined ? global$1.TYPED_ARRAY_SUPPORT : true; +/* + * Export kMaxLength after typed array support is determined. + */ +kMaxLength(); + function kMaxLength () { return Buffer.TYPED_ARRAY_SUPPORT ? 0x7fffffff @@ -324,6 +329,8 @@ Buffer.from = function (value, encodingOrOffset, length) { if (Buffer.TYPED_ARRAY_SUPPORT) { Buffer.prototype.__proto__ = Uint8Array.prototype; Buffer.__proto__ = Uint8Array; + if (typeof Symbol !== 'undefined' && Symbol.species && + Buffer[Symbol.species] === Buffer) ; } function assertSize (size) { diff --git a/packages/csv-stringify/dist/iife/index.js b/packages/csv-stringify/dist/iife/index.js index 8331c85ed..c91f1b60e 100644 --- a/packages/csv-stringify/dist/iife/index.js +++ b/packages/csv-stringify/dist/iife/index.js @@ -702,6 +702,11 @@ var csv_stringify = (function (exports) { ? global$1.TYPED_ARRAY_SUPPORT : true; + /* + * Export kMaxLength after typed array support is determined. + */ + kMaxLength(); + function kMaxLength () { return Buffer.TYPED_ARRAY_SUPPORT ? 0x7fffffff @@ -793,6 +798,8 @@ var csv_stringify = (function (exports) { if (Buffer.TYPED_ARRAY_SUPPORT) { Buffer.prototype.__proto__ = Uint8Array.prototype; Buffer.__proto__ = Uint8Array; + if (typeof Symbol !== 'undefined' && Symbol.species && + Buffer[Symbol.species] === Buffer) ; } function assertSize (size) { @@ -2565,16 +2572,95 @@ var csv_stringify = (function (exports) { Item.prototype.run = function () { this.fun.apply(null, this.array); }; + var title = 'browser'; + var platform = 'browser'; + var browser = true; + var env = {}; + var argv = []; + var version = ''; // empty string to avoid regexp issues + var versions = {}; + var release = {}; + var config = {}; + + function noop() {} + + var on = noop; + var addListener = noop; + var once = noop; + var off = noop; + var removeListener = noop; + var removeAllListeners = noop; + var emit = noop; + + function binding(name) { + throw new Error('process.binding is not supported'); + } + + function cwd () { return '/' } + function chdir (dir) { + throw new Error('process.chdir is not supported'); + }function umask() { return 0; } // from https://github.com/kumavis/browser-process-hrtime/blob/master/index.js var performance = global$1.performance || {}; - performance.now || + var performanceNow = + performance.now || performance.mozNow || performance.msNow || performance.oNow || performance.webkitNow || function(){ return (new Date()).getTime() }; + // generate timestamp or delta + // see http://nodejs.org/api/process.html#process_process_hrtime + function hrtime(previousTimestamp){ + var clocktime = performanceNow.call(performance)*1e-3; + var seconds = Math.floor(clocktime); + var nanoseconds = Math.floor((clocktime%1)*1e9); + if (previousTimestamp) { + seconds = seconds - previousTimestamp[0]; + nanoseconds = nanoseconds - previousTimestamp[1]; + if (nanoseconds<0) { + seconds--; + nanoseconds += 1e9; + } + } + return [seconds,nanoseconds] + } + + var startTime = new Date(); + function uptime() { + var currentTime = new Date(); + var dif = currentTime - startTime; + return dif / 1000; + } + + var process = { + nextTick: nextTick, + title: title, + browser: browser, + env: env, + argv: argv, + version: version, + versions: versions, + on: on, + addListener: addListener, + once: once, + off: off, + removeListener: removeListener, + removeAllListeners: removeAllListeners, + emit: emit, + binding: binding, + cwd: cwd, + chdir: chdir, + umask: umask, + hrtime: hrtime, + platform: platform, + release: release, + config: config, + uptime: uptime + }; + var inherits; if (typeof Object.create === 'function'){ inherits = function inherits(ctor, superCtor) { @@ -2650,10 +2736,18 @@ var csv_stringify = (function (exports) { }; } + if (process.noDeprecation === true) { + return fn; + } + var warned = false; function deprecated() { if (!warned) { - { + if (process.throwDeprecation) { + throw new Error(msg); + } else if (process.traceDeprecation) { + console.trace(msg); + } else { console.error(msg); } warned = true; @@ -2668,7 +2762,7 @@ var csv_stringify = (function (exports) { var debugEnviron; function debuglog(set) { if (isUndefined(debugEnviron)) - debugEnviron = ''; + debugEnviron = process.env.NODE_DEBUG || ''; set = set.toUpperCase(); if (!debugs[set]) { if (new RegExp('\\b' + set + '\\b', 'i').test(debugEnviron)) { diff --git a/packages/csv-stringify/dist/iife/sync.js b/packages/csv-stringify/dist/iife/sync.js index ac4afe5d6..414e5f674 100644 --- a/packages/csv-stringify/dist/iife/sync.js +++ b/packages/csv-stringify/dist/iife/sync.js @@ -236,6 +236,11 @@ var csv_stringify_sync = (function (exports) { ? global$1.TYPED_ARRAY_SUPPORT : true; + /* + * Export kMaxLength after typed array support is determined. + */ + kMaxLength(); + function kMaxLength () { return Buffer.TYPED_ARRAY_SUPPORT ? 0x7fffffff @@ -327,6 +332,8 @@ var csv_stringify_sync = (function (exports) { if (Buffer.TYPED_ARRAY_SUPPORT) { Buffer.prototype.__proto__ = Uint8Array.prototype; Buffer.__proto__ = Uint8Array; + if (typeof Symbol !== 'undefined' && Symbol.species && + Buffer[Symbol.species] === Buffer) ; } function assertSize (size) { diff --git a/packages/csv-stringify/dist/umd/index.js b/packages/csv-stringify/dist/umd/index.js index 35ac97df1..b31694836 100644 --- a/packages/csv-stringify/dist/umd/index.js +++ b/packages/csv-stringify/dist/umd/index.js @@ -705,6 +705,11 @@ ? global$1.TYPED_ARRAY_SUPPORT : true; + /* + * Export kMaxLength after typed array support is determined. + */ + kMaxLength(); + function kMaxLength () { return Buffer.TYPED_ARRAY_SUPPORT ? 0x7fffffff @@ -796,6 +801,8 @@ if (Buffer.TYPED_ARRAY_SUPPORT) { Buffer.prototype.__proto__ = Uint8Array.prototype; Buffer.__proto__ = Uint8Array; + if (typeof Symbol !== 'undefined' && Symbol.species && + Buffer[Symbol.species] === Buffer) ; } function assertSize (size) { @@ -2568,16 +2575,95 @@ Item.prototype.run = function () { this.fun.apply(null, this.array); }; + var title = 'browser'; + var platform = 'browser'; + var browser = true; + var env = {}; + var argv = []; + var version = ''; // empty string to avoid regexp issues + var versions = {}; + var release = {}; + var config = {}; + + function noop() {} + + var on = noop; + var addListener = noop; + var once = noop; + var off = noop; + var removeListener = noop; + var removeAllListeners = noop; + var emit = noop; + + function binding(name) { + throw new Error('process.binding is not supported'); + } + + function cwd () { return '/' } + function chdir (dir) { + throw new Error('process.chdir is not supported'); + }function umask() { return 0; } // from https://github.com/kumavis/browser-process-hrtime/blob/master/index.js var performance = global$1.performance || {}; - performance.now || + var performanceNow = + performance.now || performance.mozNow || performance.msNow || performance.oNow || performance.webkitNow || function(){ return (new Date()).getTime() }; + // generate timestamp or delta + // see http://nodejs.org/api/process.html#process_process_hrtime + function hrtime(previousTimestamp){ + var clocktime = performanceNow.call(performance)*1e-3; + var seconds = Math.floor(clocktime); + var nanoseconds = Math.floor((clocktime%1)*1e9); + if (previousTimestamp) { + seconds = seconds - previousTimestamp[0]; + nanoseconds = nanoseconds - previousTimestamp[1]; + if (nanoseconds<0) { + seconds--; + nanoseconds += 1e9; + } + } + return [seconds,nanoseconds] + } + + var startTime = new Date(); + function uptime() { + var currentTime = new Date(); + var dif = currentTime - startTime; + return dif / 1000; + } + + var process = { + nextTick: nextTick, + title: title, + browser: browser, + env: env, + argv: argv, + version: version, + versions: versions, + on: on, + addListener: addListener, + once: once, + off: off, + removeListener: removeListener, + removeAllListeners: removeAllListeners, + emit: emit, + binding: binding, + cwd: cwd, + chdir: chdir, + umask: umask, + hrtime: hrtime, + platform: platform, + release: release, + config: config, + uptime: uptime + }; + var inherits; if (typeof Object.create === 'function'){ inherits = function inherits(ctor, superCtor) { @@ -2653,10 +2739,18 @@ }; } + if (process.noDeprecation === true) { + return fn; + } + var warned = false; function deprecated() { if (!warned) { - { + if (process.throwDeprecation) { + throw new Error(msg); + } else if (process.traceDeprecation) { + console.trace(msg); + } else { console.error(msg); } warned = true; @@ -2671,7 +2765,7 @@ var debugEnviron; function debuglog(set) { if (isUndefined(debugEnviron)) - debugEnviron = ''; + debugEnviron = process.env.NODE_DEBUG || ''; set = set.toUpperCase(); if (!debugs[set]) { if (new RegExp('\\b' + set + '\\b', 'i').test(debugEnviron)) { diff --git a/packages/csv-stringify/dist/umd/sync.js b/packages/csv-stringify/dist/umd/sync.js index 1db0ff077..3376f0c50 100644 --- a/packages/csv-stringify/dist/umd/sync.js +++ b/packages/csv-stringify/dist/umd/sync.js @@ -239,6 +239,11 @@ ? global$1.TYPED_ARRAY_SUPPORT : true; + /* + * Export kMaxLength after typed array support is determined. + */ + kMaxLength(); + function kMaxLength () { return Buffer.TYPED_ARRAY_SUPPORT ? 0x7fffffff @@ -330,6 +335,8 @@ if (Buffer.TYPED_ARRAY_SUPPORT) { Buffer.prototype.__proto__ = Uint8Array.prototype; Buffer.__proto__ = Uint8Array; + if (typeof Symbol !== 'undefined' && Symbol.species && + Buffer[Symbol.species] === Buffer) ; } function assertSize (size) { diff --git a/packages/csv-stringify/package.json b/packages/csv-stringify/package.json index ffd9070c4..30783560f 100644 --- a/packages/csv-stringify/package.json +++ b/packages/csv-stringify/package.json @@ -9,23 +9,23 @@ ], "author": "David Worms (https://www.adaltas.com)", "devDependencies": { - "@rollup/plugin-eslint": "^8.0.1", - "@rollup/plugin-node-resolve": "^13.1.3", - "@types/mocha": "^9.1.0", - "@types/node": "^17.0.18", + "@rollup/plugin-eslint": "^8.0.2", + "@rollup/plugin-node-resolve": "^13.3.0", + "@types/mocha": "^9.1.1", + "@types/node": "^17.0.35", "@types/should": "^13.0.0", - "coffeescript": "~2.6.1", + "coffeescript": "~2.7.0", "csv-generate": "^4.0.4", "each": "^1.2.2", - "eslint": "^8.9.0", - "express": "^4.17.2", - "mocha": "~9.2.0", - "rollup": "^2.67.2", + "eslint": "^8.16.0", + "express": "^4.18.1", + "mocha": "~10.0.0", + "rollup": "^2.74.1", "rollup-plugin-node-builtins": "^2.1.2", "rollup-plugin-node-globals": "^1.4.0", "should": "~13.2.3", - "ts-node": "^10.5.0", - "typescript": "^4.5.5" + "ts-node": "^10.8.0", + "typescript": "^4.6.4" }, "exports": { ".": { diff --git a/packages/csv/dist/cjs/index.cjs b/packages/csv/dist/cjs/index.cjs index 3f9fbc8a6..b99808a9b 100644 --- a/packages/csv/dist/cjs/index.cjs +++ b/packages/csv/dist/cjs/index.cjs @@ -138,11 +138,11 @@ const read = (options, state, size, push, close) => { // Create the record let record = []; let recordLength; - options.columns.forEach((fn) => { + for(const fn of options.columns){ const result = fn({options: options, state: state}); const type = typeof result; if(result !== null && type !== 'string' && type !== 'number'){ - throw Error([ + return Error([ 'INVALID_VALUE:', 'values returned by column function must be', 'a string, a number or null,', @@ -150,7 +150,7 @@ const read = (options, state, size, push, close) => { ].join(' ')); } record.push(result); - }); + } // Obtain record length if(options.objectMode){ recordLength = 0; @@ -203,11 +203,14 @@ Generator.prototype.end = function(){ // Put new data into the read queue. Generator.prototype._read = function(size){ const self = this; - read(this.options, this.state, size, function(chunk) { + const err = read(this.options, this.state, size, function(chunk) { self.__push(chunk); }, function(){ self.push(null); }); + if(err){ + this.destroy(err); + } }; // Put new data into the read queue. Generator.prototype.__push = function(record){ diff --git a/packages/csv/dist/cjs/sync.cjs b/packages/csv/dist/cjs/sync.cjs index 62dc2e86f..a4ebf619f 100644 --- a/packages/csv/dist/cjs/sync.cjs +++ b/packages/csv/dist/cjs/sync.cjs @@ -138,11 +138,11 @@ const read = (options, state, size, push, close) => { // Create the record let record = []; let recordLength; - options.columns.forEach((fn) => { + for(const fn of options.columns){ const result = fn({options: options, state: state}); const type = typeof result; if(result !== null && type !== 'string' && type !== 'number'){ - throw Error([ + return Error([ 'INVALID_VALUE:', 'values returned by column function must be', 'a string, a number or null,', @@ -150,7 +150,7 @@ const read = (options, state, size, push, close) => { ].join(' ')); } record.push(result); - }); + } // Obtain record length if(options.objectMode){ recordLength = 0; @@ -203,11 +203,14 @@ Generator.prototype.end = function(){ // Put new data into the read queue. Generator.prototype._read = function(size){ const self = this; - read(this.options, this.state, size, function(chunk) { + const err = read(this.options, this.state, size, function(chunk) { self.__push(chunk); }, function(){ self.push(null); }); + if(err){ + this.destroy(err); + } }; // Put new data into the read queue. Generator.prototype.__push = function(record){ diff --git a/packages/csv/dist/esm/index.js b/packages/csv/dist/esm/index.js index 28939ae86..dff108911 100644 --- a/packages/csv/dist/esm/index.js +++ b/packages/csv/dist/esm/index.js @@ -233,6 +233,11 @@ Buffer.TYPED_ARRAY_SUPPORT = global$1.TYPED_ARRAY_SUPPORT !== undefined ? global$1.TYPED_ARRAY_SUPPORT : true; +/* + * Export kMaxLength after typed array support is determined. + */ +kMaxLength(); + function kMaxLength () { return Buffer.TYPED_ARRAY_SUPPORT ? 0x7fffffff @@ -324,6 +329,8 @@ Buffer.from = function (value, encodingOrOffset, length) { if (Buffer.TYPED_ARRAY_SUPPORT) { Buffer.prototype.__proto__ = Uint8Array.prototype; Buffer.__proto__ = Uint8Array; + if (typeof Symbol !== 'undefined' && Symbol.species && + Buffer[Symbol.species] === Buffer) ; } function assertSize (size) { @@ -2562,16 +2569,95 @@ function Item(fun, array) { Item.prototype.run = function () { this.fun.apply(null, this.array); }; +var title = 'browser'; +var platform = 'browser'; +var browser = true; +var env = {}; +var argv = []; +var version = ''; // empty string to avoid regexp issues +var versions = {}; +var release = {}; +var config = {}; + +function noop() {} + +var on = noop; +var addListener = noop; +var once = noop; +var off = noop; +var removeListener = noop; +var removeAllListeners = noop; +var emit = noop; + +function binding(name) { + throw new Error('process.binding is not supported'); +} + +function cwd () { return '/' } +function chdir (dir) { + throw new Error('process.chdir is not supported'); +}function umask() { return 0; } // from https://github.com/kumavis/browser-process-hrtime/blob/master/index.js var performance = global$1.performance || {}; -performance.now || +var performanceNow = + performance.now || performance.mozNow || performance.msNow || performance.oNow || performance.webkitNow || function(){ return (new Date()).getTime() }; +// generate timestamp or delta +// see http://nodejs.org/api/process.html#process_process_hrtime +function hrtime(previousTimestamp){ + var clocktime = performanceNow.call(performance)*1e-3; + var seconds = Math.floor(clocktime); + var nanoseconds = Math.floor((clocktime%1)*1e9); + if (previousTimestamp) { + seconds = seconds - previousTimestamp[0]; + nanoseconds = nanoseconds - previousTimestamp[1]; + if (nanoseconds<0) { + seconds--; + nanoseconds += 1e9; + } + } + return [seconds,nanoseconds] +} + +var startTime = new Date(); +function uptime() { + var currentTime = new Date(); + var dif = currentTime - startTime; + return dif / 1000; +} + +var process = { + nextTick: nextTick, + title: title, + browser: browser, + env: env, + argv: argv, + version: version, + versions: versions, + on: on, + addListener: addListener, + once: once, + off: off, + removeListener: removeListener, + removeAllListeners: removeAllListeners, + emit: emit, + binding: binding, + cwd: cwd, + chdir: chdir, + umask: umask, + hrtime: hrtime, + platform: platform, + release: release, + config: config, + uptime: uptime +}; + var inherits; if (typeof Object.create === 'function'){ inherits = function inherits(ctor, superCtor) { @@ -2647,10 +2733,18 @@ function deprecate(fn, msg) { }; } + if (process.noDeprecation === true) { + return fn; + } + var warned = false; function deprecated() { if (!warned) { - { + if (process.throwDeprecation) { + throw new Error(msg); + } else if (process.traceDeprecation) { + console.trace(msg); + } else { console.error(msg); } warned = true; @@ -2665,7 +2759,7 @@ var debugs = {}; var debugEnviron; function debuglog(set) { if (isUndefined(debugEnviron)) - debugEnviron = ''; + debugEnviron = process.env.NODE_DEBUG || ''; set = set.toUpperCase(); if (!debugs[set]) { if (new RegExp('\\b' + set + '\\b', 'i').test(debugEnviron)) { @@ -5164,11 +5258,11 @@ const read = (options, state, size, push, close) => { // Create the record let record = []; let recordLength; - options.columns.forEach((fn) => { + for(const fn of options.columns){ const result = fn({options: options, state: state}); const type = typeof result; if(result !== null && type !== 'string' && type !== 'number'){ - throw Error([ + return Error([ 'INVALID_VALUE:', 'values returned by column function must be', 'a string, a number or null,', @@ -5176,7 +5270,7 @@ const read = (options, state, size, push, close) => { ].join(' ')); } record.push(result); - }); + } // Obtain record length if(options.objectMode){ recordLength = 0; @@ -5229,11 +5323,14 @@ Generator.prototype.end = function(){ // Put new data into the read queue. Generator.prototype._read = function(size){ const self = this; - read(this.options, this.state, size, function(chunk) { + const err = read(this.options, this.state, size, function(chunk) { self.__push(chunk); }, function(){ self.push(null); }); + if(err){ + this.destroy(err); + } }; // Put new data into the read queue. Generator.prototype.__push = function(record){ diff --git a/packages/csv/dist/esm/sync.js b/packages/csv/dist/esm/sync.js index c438b78e7..80109b689 100644 --- a/packages/csv/dist/esm/sync.js +++ b/packages/csv/dist/esm/sync.js @@ -233,6 +233,11 @@ Buffer.TYPED_ARRAY_SUPPORT = global$1.TYPED_ARRAY_SUPPORT !== undefined ? global$1.TYPED_ARRAY_SUPPORT : true; +/* + * Export kMaxLength after typed array support is determined. + */ +kMaxLength(); + function kMaxLength () { return Buffer.TYPED_ARRAY_SUPPORT ? 0x7fffffff @@ -324,6 +329,8 @@ Buffer.from = function (value, encodingOrOffset, length) { if (Buffer.TYPED_ARRAY_SUPPORT) { Buffer.prototype.__proto__ = Uint8Array.prototype; Buffer.__proto__ = Uint8Array; + if (typeof Symbol !== 'undefined' && Symbol.species && + Buffer[Symbol.species] === Buffer) ; } function assertSize (size) { @@ -2562,16 +2569,95 @@ function Item(fun, array) { Item.prototype.run = function () { this.fun.apply(null, this.array); }; +var title = 'browser'; +var platform = 'browser'; +var browser = true; +var env = {}; +var argv = []; +var version = ''; // empty string to avoid regexp issues +var versions = {}; +var release = {}; +var config = {}; + +function noop() {} + +var on = noop; +var addListener = noop; +var once = noop; +var off = noop; +var removeListener = noop; +var removeAllListeners = noop; +var emit = noop; + +function binding(name) { + throw new Error('process.binding is not supported'); +} + +function cwd () { return '/' } +function chdir (dir) { + throw new Error('process.chdir is not supported'); +}function umask() { return 0; } // from https://github.com/kumavis/browser-process-hrtime/blob/master/index.js var performance = global$1.performance || {}; -performance.now || +var performanceNow = + performance.now || performance.mozNow || performance.msNow || performance.oNow || performance.webkitNow || function(){ return (new Date()).getTime() }; +// generate timestamp or delta +// see http://nodejs.org/api/process.html#process_process_hrtime +function hrtime(previousTimestamp){ + var clocktime = performanceNow.call(performance)*1e-3; + var seconds = Math.floor(clocktime); + var nanoseconds = Math.floor((clocktime%1)*1e9); + if (previousTimestamp) { + seconds = seconds - previousTimestamp[0]; + nanoseconds = nanoseconds - previousTimestamp[1]; + if (nanoseconds<0) { + seconds--; + nanoseconds += 1e9; + } + } + return [seconds,nanoseconds] +} + +var startTime = new Date(); +function uptime() { + var currentTime = new Date(); + var dif = currentTime - startTime; + return dif / 1000; +} + +var process = { + nextTick: nextTick, + title: title, + browser: browser, + env: env, + argv: argv, + version: version, + versions: versions, + on: on, + addListener: addListener, + once: once, + off: off, + removeListener: removeListener, + removeAllListeners: removeAllListeners, + emit: emit, + binding: binding, + cwd: cwd, + chdir: chdir, + umask: umask, + hrtime: hrtime, + platform: platform, + release: release, + config: config, + uptime: uptime +}; + var inherits; if (typeof Object.create === 'function'){ inherits = function inherits(ctor, superCtor) { @@ -2647,10 +2733,18 @@ function deprecate(fn, msg) { }; } + if (process.noDeprecation === true) { + return fn; + } + var warned = false; function deprecated() { if (!warned) { - { + if (process.throwDeprecation) { + throw new Error(msg); + } else if (process.traceDeprecation) { + console.trace(msg); + } else { console.error(msg); } warned = true; @@ -2665,7 +2759,7 @@ var debugs = {}; var debugEnviron; function debuglog(set) { if (isUndefined(debugEnviron)) - debugEnviron = ''; + debugEnviron = process.env.NODE_DEBUG || ''; set = set.toUpperCase(); if (!debugs[set]) { if (new RegExp('\\b' + set + '\\b', 'i').test(debugEnviron)) { @@ -5164,11 +5258,11 @@ const read = (options, state, size, push, close) => { // Create the record let record = []; let recordLength; - options.columns.forEach((fn) => { + for(const fn of options.columns){ const result = fn({options: options, state: state}); const type = typeof result; if(result !== null && type !== 'string' && type !== 'number'){ - throw Error([ + return Error([ 'INVALID_VALUE:', 'values returned by column function must be', 'a string, a number or null,', @@ -5176,7 +5270,7 @@ const read = (options, state, size, push, close) => { ].join(' ')); } record.push(result); - }); + } // Obtain record length if(options.objectMode){ recordLength = 0; @@ -5229,11 +5323,14 @@ Generator.prototype.end = function(){ // Put new data into the read queue. Generator.prototype._read = function(size){ const self = this; - read(this.options, this.state, size, function(chunk) { + const err = read(this.options, this.state, size, function(chunk) { self.__push(chunk); }, function(){ self.push(null); }); + if(err){ + this.destroy(err); + } }; // Put new data into the read queue. Generator.prototype.__push = function(record){ diff --git a/packages/csv/dist/iife/index.js b/packages/csv/dist/iife/index.js index b21cd9b47..48d3cfb93 100644 --- a/packages/csv/dist/iife/index.js +++ b/packages/csv/dist/iife/index.js @@ -236,6 +236,11 @@ var csv = (function (exports) { ? global$1.TYPED_ARRAY_SUPPORT : true; + /* + * Export kMaxLength after typed array support is determined. + */ + kMaxLength(); + function kMaxLength () { return Buffer.TYPED_ARRAY_SUPPORT ? 0x7fffffff @@ -327,6 +332,8 @@ var csv = (function (exports) { if (Buffer.TYPED_ARRAY_SUPPORT) { Buffer.prototype.__proto__ = Uint8Array.prototype; Buffer.__proto__ = Uint8Array; + if (typeof Symbol !== 'undefined' && Symbol.species && + Buffer[Symbol.species] === Buffer) ; } function assertSize (size) { @@ -2565,16 +2572,95 @@ var csv = (function (exports) { Item.prototype.run = function () { this.fun.apply(null, this.array); }; + var title = 'browser'; + var platform = 'browser'; + var browser = true; + var env = {}; + var argv = []; + var version = ''; // empty string to avoid regexp issues + var versions = {}; + var release = {}; + var config = {}; + + function noop() {} + + var on = noop; + var addListener = noop; + var once = noop; + var off = noop; + var removeListener = noop; + var removeAllListeners = noop; + var emit = noop; + + function binding(name) { + throw new Error('process.binding is not supported'); + } + + function cwd () { return '/' } + function chdir (dir) { + throw new Error('process.chdir is not supported'); + }function umask() { return 0; } // from https://github.com/kumavis/browser-process-hrtime/blob/master/index.js var performance = global$1.performance || {}; - performance.now || + var performanceNow = + performance.now || performance.mozNow || performance.msNow || performance.oNow || performance.webkitNow || function(){ return (new Date()).getTime() }; + // generate timestamp or delta + // see http://nodejs.org/api/process.html#process_process_hrtime + function hrtime(previousTimestamp){ + var clocktime = performanceNow.call(performance)*1e-3; + var seconds = Math.floor(clocktime); + var nanoseconds = Math.floor((clocktime%1)*1e9); + if (previousTimestamp) { + seconds = seconds - previousTimestamp[0]; + nanoseconds = nanoseconds - previousTimestamp[1]; + if (nanoseconds<0) { + seconds--; + nanoseconds += 1e9; + } + } + return [seconds,nanoseconds] + } + + var startTime = new Date(); + function uptime() { + var currentTime = new Date(); + var dif = currentTime - startTime; + return dif / 1000; + } + + var process = { + nextTick: nextTick, + title: title, + browser: browser, + env: env, + argv: argv, + version: version, + versions: versions, + on: on, + addListener: addListener, + once: once, + off: off, + removeListener: removeListener, + removeAllListeners: removeAllListeners, + emit: emit, + binding: binding, + cwd: cwd, + chdir: chdir, + umask: umask, + hrtime: hrtime, + platform: platform, + release: release, + config: config, + uptime: uptime + }; + var inherits; if (typeof Object.create === 'function'){ inherits = function inherits(ctor, superCtor) { @@ -2650,10 +2736,18 @@ var csv = (function (exports) { }; } + if (process.noDeprecation === true) { + return fn; + } + var warned = false; function deprecated() { if (!warned) { - { + if (process.throwDeprecation) { + throw new Error(msg); + } else if (process.traceDeprecation) { + console.trace(msg); + } else { console.error(msg); } warned = true; @@ -2668,7 +2762,7 @@ var csv = (function (exports) { var debugEnviron; function debuglog(set) { if (isUndefined(debugEnviron)) - debugEnviron = ''; + debugEnviron = process.env.NODE_DEBUG || ''; set = set.toUpperCase(); if (!debugs[set]) { if (new RegExp('\\b' + set + '\\b', 'i').test(debugEnviron)) { @@ -5167,11 +5261,11 @@ var csv = (function (exports) { // Create the record let record = []; let recordLength; - options.columns.forEach((fn) => { + for(const fn of options.columns){ const result = fn({options: options, state: state}); const type = typeof result; if(result !== null && type !== 'string' && type !== 'number'){ - throw Error([ + return Error([ 'INVALID_VALUE:', 'values returned by column function must be', 'a string, a number or null,', @@ -5179,7 +5273,7 @@ var csv = (function (exports) { ].join(' ')); } record.push(result); - }); + } // Obtain record length if(options.objectMode){ recordLength = 0; @@ -5232,11 +5326,14 @@ var csv = (function (exports) { // Put new data into the read queue. Generator.prototype._read = function(size){ const self = this; - read(this.options, this.state, size, function(chunk) { + const err = read(this.options, this.state, size, function(chunk) { self.__push(chunk); }, function(){ self.push(null); }); + if(err){ + this.destroy(err); + } }; // Put new data into the read queue. Generator.prototype.__push = function(record){ diff --git a/packages/csv/dist/iife/sync.js b/packages/csv/dist/iife/sync.js index c3696d179..f93108f74 100644 --- a/packages/csv/dist/iife/sync.js +++ b/packages/csv/dist/iife/sync.js @@ -236,6 +236,11 @@ var csv_sync = (function (exports) { ? global$1.TYPED_ARRAY_SUPPORT : true; + /* + * Export kMaxLength after typed array support is determined. + */ + kMaxLength(); + function kMaxLength () { return Buffer.TYPED_ARRAY_SUPPORT ? 0x7fffffff @@ -327,6 +332,8 @@ var csv_sync = (function (exports) { if (Buffer.TYPED_ARRAY_SUPPORT) { Buffer.prototype.__proto__ = Uint8Array.prototype; Buffer.__proto__ = Uint8Array; + if (typeof Symbol !== 'undefined' && Symbol.species && + Buffer[Symbol.species] === Buffer) ; } function assertSize (size) { @@ -2565,16 +2572,95 @@ var csv_sync = (function (exports) { Item.prototype.run = function () { this.fun.apply(null, this.array); }; + var title = 'browser'; + var platform = 'browser'; + var browser = true; + var env = {}; + var argv = []; + var version = ''; // empty string to avoid regexp issues + var versions = {}; + var release = {}; + var config = {}; + + function noop() {} + + var on = noop; + var addListener = noop; + var once = noop; + var off = noop; + var removeListener = noop; + var removeAllListeners = noop; + var emit = noop; + + function binding(name) { + throw new Error('process.binding is not supported'); + } + + function cwd () { return '/' } + function chdir (dir) { + throw new Error('process.chdir is not supported'); + }function umask() { return 0; } // from https://github.com/kumavis/browser-process-hrtime/blob/master/index.js var performance = global$1.performance || {}; - performance.now || + var performanceNow = + performance.now || performance.mozNow || performance.msNow || performance.oNow || performance.webkitNow || function(){ return (new Date()).getTime() }; + // generate timestamp or delta + // see http://nodejs.org/api/process.html#process_process_hrtime + function hrtime(previousTimestamp){ + var clocktime = performanceNow.call(performance)*1e-3; + var seconds = Math.floor(clocktime); + var nanoseconds = Math.floor((clocktime%1)*1e9); + if (previousTimestamp) { + seconds = seconds - previousTimestamp[0]; + nanoseconds = nanoseconds - previousTimestamp[1]; + if (nanoseconds<0) { + seconds--; + nanoseconds += 1e9; + } + } + return [seconds,nanoseconds] + } + + var startTime = new Date(); + function uptime() { + var currentTime = new Date(); + var dif = currentTime - startTime; + return dif / 1000; + } + + var process = { + nextTick: nextTick, + title: title, + browser: browser, + env: env, + argv: argv, + version: version, + versions: versions, + on: on, + addListener: addListener, + once: once, + off: off, + removeListener: removeListener, + removeAllListeners: removeAllListeners, + emit: emit, + binding: binding, + cwd: cwd, + chdir: chdir, + umask: umask, + hrtime: hrtime, + platform: platform, + release: release, + config: config, + uptime: uptime + }; + var inherits; if (typeof Object.create === 'function'){ inherits = function inherits(ctor, superCtor) { @@ -2650,10 +2736,18 @@ var csv_sync = (function (exports) { }; } + if (process.noDeprecation === true) { + return fn; + } + var warned = false; function deprecated() { if (!warned) { - { + if (process.throwDeprecation) { + throw new Error(msg); + } else if (process.traceDeprecation) { + console.trace(msg); + } else { console.error(msg); } warned = true; @@ -2668,7 +2762,7 @@ var csv_sync = (function (exports) { var debugEnviron; function debuglog(set) { if (isUndefined(debugEnviron)) - debugEnviron = ''; + debugEnviron = process.env.NODE_DEBUG || ''; set = set.toUpperCase(); if (!debugs[set]) { if (new RegExp('\\b' + set + '\\b', 'i').test(debugEnviron)) { @@ -5167,11 +5261,11 @@ var csv_sync = (function (exports) { // Create the record let record = []; let recordLength; - options.columns.forEach((fn) => { + for(const fn of options.columns){ const result = fn({options: options, state: state}); const type = typeof result; if(result !== null && type !== 'string' && type !== 'number'){ - throw Error([ + return Error([ 'INVALID_VALUE:', 'values returned by column function must be', 'a string, a number or null,', @@ -5179,7 +5273,7 @@ var csv_sync = (function (exports) { ].join(' ')); } record.push(result); - }); + } // Obtain record length if(options.objectMode){ recordLength = 0; @@ -5232,11 +5326,14 @@ var csv_sync = (function (exports) { // Put new data into the read queue. Generator.prototype._read = function(size){ const self = this; - read(this.options, this.state, size, function(chunk) { + const err = read(this.options, this.state, size, function(chunk) { self.__push(chunk); }, function(){ self.push(null); }); + if(err){ + this.destroy(err); + } }; // Put new data into the read queue. Generator.prototype.__push = function(record){ diff --git a/packages/csv/dist/umd/index.js b/packages/csv/dist/umd/index.js index b72ab5fd3..6c83b4488 100644 --- a/packages/csv/dist/umd/index.js +++ b/packages/csv/dist/umd/index.js @@ -239,6 +239,11 @@ ? global$1.TYPED_ARRAY_SUPPORT : true; + /* + * Export kMaxLength after typed array support is determined. + */ + kMaxLength(); + function kMaxLength () { return Buffer.TYPED_ARRAY_SUPPORT ? 0x7fffffff @@ -330,6 +335,8 @@ if (Buffer.TYPED_ARRAY_SUPPORT) { Buffer.prototype.__proto__ = Uint8Array.prototype; Buffer.__proto__ = Uint8Array; + if (typeof Symbol !== 'undefined' && Symbol.species && + Buffer[Symbol.species] === Buffer) ; } function assertSize (size) { @@ -2568,16 +2575,95 @@ Item.prototype.run = function () { this.fun.apply(null, this.array); }; + var title = 'browser'; + var platform = 'browser'; + var browser = true; + var env = {}; + var argv = []; + var version = ''; // empty string to avoid regexp issues + var versions = {}; + var release = {}; + var config = {}; + + function noop() {} + + var on = noop; + var addListener = noop; + var once = noop; + var off = noop; + var removeListener = noop; + var removeAllListeners = noop; + var emit = noop; + + function binding(name) { + throw new Error('process.binding is not supported'); + } + + function cwd () { return '/' } + function chdir (dir) { + throw new Error('process.chdir is not supported'); + }function umask() { return 0; } // from https://github.com/kumavis/browser-process-hrtime/blob/master/index.js var performance = global$1.performance || {}; - performance.now || + var performanceNow = + performance.now || performance.mozNow || performance.msNow || performance.oNow || performance.webkitNow || function(){ return (new Date()).getTime() }; + // generate timestamp or delta + // see http://nodejs.org/api/process.html#process_process_hrtime + function hrtime(previousTimestamp){ + var clocktime = performanceNow.call(performance)*1e-3; + var seconds = Math.floor(clocktime); + var nanoseconds = Math.floor((clocktime%1)*1e9); + if (previousTimestamp) { + seconds = seconds - previousTimestamp[0]; + nanoseconds = nanoseconds - previousTimestamp[1]; + if (nanoseconds<0) { + seconds--; + nanoseconds += 1e9; + } + } + return [seconds,nanoseconds] + } + + var startTime = new Date(); + function uptime() { + var currentTime = new Date(); + var dif = currentTime - startTime; + return dif / 1000; + } + + var process = { + nextTick: nextTick, + title: title, + browser: browser, + env: env, + argv: argv, + version: version, + versions: versions, + on: on, + addListener: addListener, + once: once, + off: off, + removeListener: removeListener, + removeAllListeners: removeAllListeners, + emit: emit, + binding: binding, + cwd: cwd, + chdir: chdir, + umask: umask, + hrtime: hrtime, + platform: platform, + release: release, + config: config, + uptime: uptime + }; + var inherits; if (typeof Object.create === 'function'){ inherits = function inherits(ctor, superCtor) { @@ -2653,10 +2739,18 @@ }; } + if (process.noDeprecation === true) { + return fn; + } + var warned = false; function deprecated() { if (!warned) { - { + if (process.throwDeprecation) { + throw new Error(msg); + } else if (process.traceDeprecation) { + console.trace(msg); + } else { console.error(msg); } warned = true; @@ -2671,7 +2765,7 @@ var debugEnviron; function debuglog(set) { if (isUndefined(debugEnviron)) - debugEnviron = ''; + debugEnviron = process.env.NODE_DEBUG || ''; set = set.toUpperCase(); if (!debugs[set]) { if (new RegExp('\\b' + set + '\\b', 'i').test(debugEnviron)) { @@ -5170,11 +5264,11 @@ // Create the record let record = []; let recordLength; - options.columns.forEach((fn) => { + for(const fn of options.columns){ const result = fn({options: options, state: state}); const type = typeof result; if(result !== null && type !== 'string' && type !== 'number'){ - throw Error([ + return Error([ 'INVALID_VALUE:', 'values returned by column function must be', 'a string, a number or null,', @@ -5182,7 +5276,7 @@ ].join(' ')); } record.push(result); - }); + } // Obtain record length if(options.objectMode){ recordLength = 0; @@ -5235,11 +5329,14 @@ // Put new data into the read queue. Generator.prototype._read = function(size){ const self = this; - read(this.options, this.state, size, function(chunk) { + const err = read(this.options, this.state, size, function(chunk) { self.__push(chunk); }, function(){ self.push(null); }); + if(err){ + this.destroy(err); + } }; // Put new data into the read queue. Generator.prototype.__push = function(record){ diff --git a/packages/csv/dist/umd/sync.js b/packages/csv/dist/umd/sync.js index 1734a972c..499fbe994 100644 --- a/packages/csv/dist/umd/sync.js +++ b/packages/csv/dist/umd/sync.js @@ -239,6 +239,11 @@ ? global$1.TYPED_ARRAY_SUPPORT : true; + /* + * Export kMaxLength after typed array support is determined. + */ + kMaxLength(); + function kMaxLength () { return Buffer.TYPED_ARRAY_SUPPORT ? 0x7fffffff @@ -330,6 +335,8 @@ if (Buffer.TYPED_ARRAY_SUPPORT) { Buffer.prototype.__proto__ = Uint8Array.prototype; Buffer.__proto__ = Uint8Array; + if (typeof Symbol !== 'undefined' && Symbol.species && + Buffer[Symbol.species] === Buffer) ; } function assertSize (size) { @@ -2568,16 +2575,95 @@ Item.prototype.run = function () { this.fun.apply(null, this.array); }; + var title = 'browser'; + var platform = 'browser'; + var browser = true; + var env = {}; + var argv = []; + var version = ''; // empty string to avoid regexp issues + var versions = {}; + var release = {}; + var config = {}; + + function noop() {} + + var on = noop; + var addListener = noop; + var once = noop; + var off = noop; + var removeListener = noop; + var removeAllListeners = noop; + var emit = noop; + + function binding(name) { + throw new Error('process.binding is not supported'); + } + + function cwd () { return '/' } + function chdir (dir) { + throw new Error('process.chdir is not supported'); + }function umask() { return 0; } // from https://github.com/kumavis/browser-process-hrtime/blob/master/index.js var performance = global$1.performance || {}; - performance.now || + var performanceNow = + performance.now || performance.mozNow || performance.msNow || performance.oNow || performance.webkitNow || function(){ return (new Date()).getTime() }; + // generate timestamp or delta + // see http://nodejs.org/api/process.html#process_process_hrtime + function hrtime(previousTimestamp){ + var clocktime = performanceNow.call(performance)*1e-3; + var seconds = Math.floor(clocktime); + var nanoseconds = Math.floor((clocktime%1)*1e9); + if (previousTimestamp) { + seconds = seconds - previousTimestamp[0]; + nanoseconds = nanoseconds - previousTimestamp[1]; + if (nanoseconds<0) { + seconds--; + nanoseconds += 1e9; + } + } + return [seconds,nanoseconds] + } + + var startTime = new Date(); + function uptime() { + var currentTime = new Date(); + var dif = currentTime - startTime; + return dif / 1000; + } + + var process = { + nextTick: nextTick, + title: title, + browser: browser, + env: env, + argv: argv, + version: version, + versions: versions, + on: on, + addListener: addListener, + once: once, + off: off, + removeListener: removeListener, + removeAllListeners: removeAllListeners, + emit: emit, + binding: binding, + cwd: cwd, + chdir: chdir, + umask: umask, + hrtime: hrtime, + platform: platform, + release: release, + config: config, + uptime: uptime + }; + var inherits; if (typeof Object.create === 'function'){ inherits = function inherits(ctor, superCtor) { @@ -2653,10 +2739,18 @@ }; } + if (process.noDeprecation === true) { + return fn; + } + var warned = false; function deprecated() { if (!warned) { - { + if (process.throwDeprecation) { + throw new Error(msg); + } else if (process.traceDeprecation) { + console.trace(msg); + } else { console.error(msg); } warned = true; @@ -2671,7 +2765,7 @@ var debugEnviron; function debuglog(set) { if (isUndefined(debugEnviron)) - debugEnviron = ''; + debugEnviron = process.env.NODE_DEBUG || ''; set = set.toUpperCase(); if (!debugs[set]) { if (new RegExp('\\b' + set + '\\b', 'i').test(debugEnviron)) { @@ -5170,11 +5264,11 @@ // Create the record let record = []; let recordLength; - options.columns.forEach((fn) => { + for(const fn of options.columns){ const result = fn({options: options, state: state}); const type = typeof result; if(result !== null && type !== 'string' && type !== 'number'){ - throw Error([ + return Error([ 'INVALID_VALUE:', 'values returned by column function must be', 'a string, a number or null,', @@ -5182,7 +5276,7 @@ ].join(' ')); } record.push(result); - }); + } // Obtain record length if(options.objectMode){ recordLength = 0; @@ -5235,11 +5329,14 @@ // Put new data into the read queue. Generator.prototype._read = function(size){ const self = this; - read(this.options, this.state, size, function(chunk) { + const err = read(this.options, this.state, size, function(chunk) { self.__push(chunk); }, function(){ self.push(null); }); + if(err){ + this.destroy(err); + } }; // Put new data into the read queue. Generator.prototype.__push = function(record){ diff --git a/packages/csv/package.json b/packages/csv/package.json index 227c736e6..dd449752f 100644 --- a/packages/csv/package.json +++ b/packages/csv/package.json @@ -27,20 +27,20 @@ "stream-transform": "^3.0.4" }, "devDependencies": { - "@rollup/plugin-eslint": "^8.0.1", - "@rollup/plugin-node-resolve": "^13.1.3", - "@types/mocha": "^9.1.0", - "@types/node": "^17.0.18", + "@rollup/plugin-eslint": "^8.0.2", + "@rollup/plugin-node-resolve": "^13.3.0", + "@types/mocha": "^9.1.1", + "@types/node": "^17.0.35", "@types/should": "^13.0.0", - "coffeescript": "~2.6.1", - "eslint": "^8.9.0", - "mocha": "~9.2.0", - "rollup": "^2.67.2", + "coffeescript": "~2.7.0", + "eslint": "^8.16.0", + "mocha": "~10.0.0", + "rollup": "^2.74.1", "rollup-plugin-node-builtins": "^2.1.2", "rollup-plugin-node-globals": "^1.4.0", "should": "~13.2.3", - "ts-node": "^10.5.0", - "typescript": "^4.5.5" + "ts-node": "^10.8.0", + "typescript": "^4.6.4" }, "engines": { "node": ">= 0.1.90" diff --git a/packages/stream-transform/dist/esm/index.js b/packages/stream-transform/dist/esm/index.js index d8e60512f..8eff00c31 100644 --- a/packages/stream-transform/dist/esm/index.js +++ b/packages/stream-transform/dist/esm/index.js @@ -699,6 +699,11 @@ Buffer.TYPED_ARRAY_SUPPORT = global$1.TYPED_ARRAY_SUPPORT !== undefined ? global$1.TYPED_ARRAY_SUPPORT : true; +/* + * Export kMaxLength after typed array support is determined. + */ +kMaxLength(); + function kMaxLength () { return Buffer.TYPED_ARRAY_SUPPORT ? 0x7fffffff @@ -790,6 +795,8 @@ Buffer.from = function (value, encodingOrOffset, length) { if (Buffer.TYPED_ARRAY_SUPPORT) { Buffer.prototype.__proto__ = Uint8Array.prototype; Buffer.__proto__ = Uint8Array; + if (typeof Symbol !== 'undefined' && Symbol.species && + Buffer[Symbol.species] === Buffer) ; } function assertSize (size) { @@ -2562,16 +2569,95 @@ function Item(fun, array) { Item.prototype.run = function () { this.fun.apply(null, this.array); }; +var title = 'browser'; +var platform = 'browser'; +var browser = true; +var env = {}; +var argv = []; +var version = ''; // empty string to avoid regexp issues +var versions = {}; +var release = {}; +var config = {}; + +function noop() {} + +var on = noop; +var addListener = noop; +var once = noop; +var off = noop; +var removeListener = noop; +var removeAllListeners = noop; +var emit = noop; + +function binding(name) { + throw new Error('process.binding is not supported'); +} + +function cwd () { return '/' } +function chdir (dir) { + throw new Error('process.chdir is not supported'); +}function umask() { return 0; } // from https://github.com/kumavis/browser-process-hrtime/blob/master/index.js var performance = global$1.performance || {}; -performance.now || +var performanceNow = + performance.now || performance.mozNow || performance.msNow || performance.oNow || performance.webkitNow || function(){ return (new Date()).getTime() }; +// generate timestamp or delta +// see http://nodejs.org/api/process.html#process_process_hrtime +function hrtime(previousTimestamp){ + var clocktime = performanceNow.call(performance)*1e-3; + var seconds = Math.floor(clocktime); + var nanoseconds = Math.floor((clocktime%1)*1e9); + if (previousTimestamp) { + seconds = seconds - previousTimestamp[0]; + nanoseconds = nanoseconds - previousTimestamp[1]; + if (nanoseconds<0) { + seconds--; + nanoseconds += 1e9; + } + } + return [seconds,nanoseconds] +} + +var startTime = new Date(); +function uptime() { + var currentTime = new Date(); + var dif = currentTime - startTime; + return dif / 1000; +} + +var process = { + nextTick: nextTick, + title: title, + browser: browser, + env: env, + argv: argv, + version: version, + versions: versions, + on: on, + addListener: addListener, + once: once, + off: off, + removeListener: removeListener, + removeAllListeners: removeAllListeners, + emit: emit, + binding: binding, + cwd: cwd, + chdir: chdir, + umask: umask, + hrtime: hrtime, + platform: platform, + release: release, + config: config, + uptime: uptime +}; + var inherits; if (typeof Object.create === 'function'){ inherits = function inherits(ctor, superCtor) { @@ -2647,10 +2733,18 @@ function deprecate(fn, msg) { }; } + if (process.noDeprecation === true) { + return fn; + } + var warned = false; function deprecated() { if (!warned) { - { + if (process.throwDeprecation) { + throw new Error(msg); + } else if (process.traceDeprecation) { + console.trace(msg); + } else { console.error(msg); } warned = true; @@ -2665,7 +2759,7 @@ var debugs = {}; var debugEnviron; function debuglog(set) { if (isUndefined(debugEnviron)) - debugEnviron = ''; + debugEnviron = process.env.NODE_DEBUG || ''; set = set.toUpperCase(); if (!debugs[set]) { if (new RegExp('\\b' + set + '\\b', 'i').test(debugEnviron)) { diff --git a/packages/stream-transform/dist/esm/sync.js b/packages/stream-transform/dist/esm/sync.js index 17f817232..ddc16298a 100644 --- a/packages/stream-transform/dist/esm/sync.js +++ b/packages/stream-transform/dist/esm/sync.js @@ -699,6 +699,11 @@ Buffer.TYPED_ARRAY_SUPPORT = global$1.TYPED_ARRAY_SUPPORT !== undefined ? global$1.TYPED_ARRAY_SUPPORT : true; +/* + * Export kMaxLength after typed array support is determined. + */ +kMaxLength(); + function kMaxLength () { return Buffer.TYPED_ARRAY_SUPPORT ? 0x7fffffff @@ -790,6 +795,8 @@ Buffer.from = function (value, encodingOrOffset, length) { if (Buffer.TYPED_ARRAY_SUPPORT) { Buffer.prototype.__proto__ = Uint8Array.prototype; Buffer.__proto__ = Uint8Array; + if (typeof Symbol !== 'undefined' && Symbol.species && + Buffer[Symbol.species] === Buffer) ; } function assertSize (size) { @@ -2562,16 +2569,95 @@ function Item(fun, array) { Item.prototype.run = function () { this.fun.apply(null, this.array); }; +var title = 'browser'; +var platform = 'browser'; +var browser = true; +var env = {}; +var argv = []; +var version = ''; // empty string to avoid regexp issues +var versions = {}; +var release = {}; +var config = {}; + +function noop() {} + +var on = noop; +var addListener = noop; +var once = noop; +var off = noop; +var removeListener = noop; +var removeAllListeners = noop; +var emit = noop; + +function binding(name) { + throw new Error('process.binding is not supported'); +} + +function cwd () { return '/' } +function chdir (dir) { + throw new Error('process.chdir is not supported'); +}function umask() { return 0; } // from https://github.com/kumavis/browser-process-hrtime/blob/master/index.js var performance = global$1.performance || {}; -performance.now || +var performanceNow = + performance.now || performance.mozNow || performance.msNow || performance.oNow || performance.webkitNow || function(){ return (new Date()).getTime() }; +// generate timestamp or delta +// see http://nodejs.org/api/process.html#process_process_hrtime +function hrtime(previousTimestamp){ + var clocktime = performanceNow.call(performance)*1e-3; + var seconds = Math.floor(clocktime); + var nanoseconds = Math.floor((clocktime%1)*1e9); + if (previousTimestamp) { + seconds = seconds - previousTimestamp[0]; + nanoseconds = nanoseconds - previousTimestamp[1]; + if (nanoseconds<0) { + seconds--; + nanoseconds += 1e9; + } + } + return [seconds,nanoseconds] +} + +var startTime = new Date(); +function uptime() { + var currentTime = new Date(); + var dif = currentTime - startTime; + return dif / 1000; +} + +var process = { + nextTick: nextTick, + title: title, + browser: browser, + env: env, + argv: argv, + version: version, + versions: versions, + on: on, + addListener: addListener, + once: once, + off: off, + removeListener: removeListener, + removeAllListeners: removeAllListeners, + emit: emit, + binding: binding, + cwd: cwd, + chdir: chdir, + umask: umask, + hrtime: hrtime, + platform: platform, + release: release, + config: config, + uptime: uptime +}; + var inherits; if (typeof Object.create === 'function'){ inherits = function inherits(ctor, superCtor) { @@ -2647,10 +2733,18 @@ function deprecate(fn, msg) { }; } + if (process.noDeprecation === true) { + return fn; + } + var warned = false; function deprecated() { if (!warned) { - { + if (process.throwDeprecation) { + throw new Error(msg); + } else if (process.traceDeprecation) { + console.trace(msg); + } else { console.error(msg); } warned = true; @@ -2665,7 +2759,7 @@ var debugs = {}; var debugEnviron; function debuglog(set) { if (isUndefined(debugEnviron)) - debugEnviron = ''; + debugEnviron = process.env.NODE_DEBUG || ''; set = set.toUpperCase(); if (!debugs[set]) { if (new RegExp('\\b' + set + '\\b', 'i').test(debugEnviron)) { diff --git a/packages/stream-transform/dist/iife/index.js b/packages/stream-transform/dist/iife/index.js index 8fce691be..1924598f4 100644 --- a/packages/stream-transform/dist/iife/index.js +++ b/packages/stream-transform/dist/iife/index.js @@ -702,6 +702,11 @@ var stream_transform = (function (exports) { ? global$1.TYPED_ARRAY_SUPPORT : true; + /* + * Export kMaxLength after typed array support is determined. + */ + kMaxLength(); + function kMaxLength () { return Buffer.TYPED_ARRAY_SUPPORT ? 0x7fffffff @@ -793,6 +798,8 @@ var stream_transform = (function (exports) { if (Buffer.TYPED_ARRAY_SUPPORT) { Buffer.prototype.__proto__ = Uint8Array.prototype; Buffer.__proto__ = Uint8Array; + if (typeof Symbol !== 'undefined' && Symbol.species && + Buffer[Symbol.species] === Buffer) ; } function assertSize (size) { @@ -2565,16 +2572,95 @@ var stream_transform = (function (exports) { Item.prototype.run = function () { this.fun.apply(null, this.array); }; + var title = 'browser'; + var platform = 'browser'; + var browser = true; + var env = {}; + var argv = []; + var version = ''; // empty string to avoid regexp issues + var versions = {}; + var release = {}; + var config = {}; + + function noop() {} + + var on = noop; + var addListener = noop; + var once = noop; + var off = noop; + var removeListener = noop; + var removeAllListeners = noop; + var emit = noop; + + function binding(name) { + throw new Error('process.binding is not supported'); + } + + function cwd () { return '/' } + function chdir (dir) { + throw new Error('process.chdir is not supported'); + }function umask() { return 0; } // from https://github.com/kumavis/browser-process-hrtime/blob/master/index.js var performance = global$1.performance || {}; - performance.now || + var performanceNow = + performance.now || performance.mozNow || performance.msNow || performance.oNow || performance.webkitNow || function(){ return (new Date()).getTime() }; + // generate timestamp or delta + // see http://nodejs.org/api/process.html#process_process_hrtime + function hrtime(previousTimestamp){ + var clocktime = performanceNow.call(performance)*1e-3; + var seconds = Math.floor(clocktime); + var nanoseconds = Math.floor((clocktime%1)*1e9); + if (previousTimestamp) { + seconds = seconds - previousTimestamp[0]; + nanoseconds = nanoseconds - previousTimestamp[1]; + if (nanoseconds<0) { + seconds--; + nanoseconds += 1e9; + } + } + return [seconds,nanoseconds] + } + + var startTime = new Date(); + function uptime() { + var currentTime = new Date(); + var dif = currentTime - startTime; + return dif / 1000; + } + + var process = { + nextTick: nextTick, + title: title, + browser: browser, + env: env, + argv: argv, + version: version, + versions: versions, + on: on, + addListener: addListener, + once: once, + off: off, + removeListener: removeListener, + removeAllListeners: removeAllListeners, + emit: emit, + binding: binding, + cwd: cwd, + chdir: chdir, + umask: umask, + hrtime: hrtime, + platform: platform, + release: release, + config: config, + uptime: uptime + }; + var inherits; if (typeof Object.create === 'function'){ inherits = function inherits(ctor, superCtor) { @@ -2650,10 +2736,18 @@ var stream_transform = (function (exports) { }; } + if (process.noDeprecation === true) { + return fn; + } + var warned = false; function deprecated() { if (!warned) { - { + if (process.throwDeprecation) { + throw new Error(msg); + } else if (process.traceDeprecation) { + console.trace(msg); + } else { console.error(msg); } warned = true; @@ -2668,7 +2762,7 @@ var stream_transform = (function (exports) { var debugEnviron; function debuglog(set) { if (isUndefined(debugEnviron)) - debugEnviron = ''; + debugEnviron = process.env.NODE_DEBUG || ''; set = set.toUpperCase(); if (!debugs[set]) { if (new RegExp('\\b' + set + '\\b', 'i').test(debugEnviron)) { diff --git a/packages/stream-transform/dist/iife/sync.js b/packages/stream-transform/dist/iife/sync.js index e2531284a..c938f2bf0 100644 --- a/packages/stream-transform/dist/iife/sync.js +++ b/packages/stream-transform/dist/iife/sync.js @@ -702,6 +702,11 @@ var stream_transform_sync = (function (exports) { ? global$1.TYPED_ARRAY_SUPPORT : true; + /* + * Export kMaxLength after typed array support is determined. + */ + kMaxLength(); + function kMaxLength () { return Buffer.TYPED_ARRAY_SUPPORT ? 0x7fffffff @@ -793,6 +798,8 @@ var stream_transform_sync = (function (exports) { if (Buffer.TYPED_ARRAY_SUPPORT) { Buffer.prototype.__proto__ = Uint8Array.prototype; Buffer.__proto__ = Uint8Array; + if (typeof Symbol !== 'undefined' && Symbol.species && + Buffer[Symbol.species] === Buffer) ; } function assertSize (size) { @@ -2565,16 +2572,95 @@ var stream_transform_sync = (function (exports) { Item.prototype.run = function () { this.fun.apply(null, this.array); }; + var title = 'browser'; + var platform = 'browser'; + var browser = true; + var env = {}; + var argv = []; + var version = ''; // empty string to avoid regexp issues + var versions = {}; + var release = {}; + var config = {}; + + function noop() {} + + var on = noop; + var addListener = noop; + var once = noop; + var off = noop; + var removeListener = noop; + var removeAllListeners = noop; + var emit = noop; + + function binding(name) { + throw new Error('process.binding is not supported'); + } + + function cwd () { return '/' } + function chdir (dir) { + throw new Error('process.chdir is not supported'); + }function umask() { return 0; } // from https://github.com/kumavis/browser-process-hrtime/blob/master/index.js var performance = global$1.performance || {}; - performance.now || + var performanceNow = + performance.now || performance.mozNow || performance.msNow || performance.oNow || performance.webkitNow || function(){ return (new Date()).getTime() }; + // generate timestamp or delta + // see http://nodejs.org/api/process.html#process_process_hrtime + function hrtime(previousTimestamp){ + var clocktime = performanceNow.call(performance)*1e-3; + var seconds = Math.floor(clocktime); + var nanoseconds = Math.floor((clocktime%1)*1e9); + if (previousTimestamp) { + seconds = seconds - previousTimestamp[0]; + nanoseconds = nanoseconds - previousTimestamp[1]; + if (nanoseconds<0) { + seconds--; + nanoseconds += 1e9; + } + } + return [seconds,nanoseconds] + } + + var startTime = new Date(); + function uptime() { + var currentTime = new Date(); + var dif = currentTime - startTime; + return dif / 1000; + } + + var process = { + nextTick: nextTick, + title: title, + browser: browser, + env: env, + argv: argv, + version: version, + versions: versions, + on: on, + addListener: addListener, + once: once, + off: off, + removeListener: removeListener, + removeAllListeners: removeAllListeners, + emit: emit, + binding: binding, + cwd: cwd, + chdir: chdir, + umask: umask, + hrtime: hrtime, + platform: platform, + release: release, + config: config, + uptime: uptime + }; + var inherits; if (typeof Object.create === 'function'){ inherits = function inherits(ctor, superCtor) { @@ -2650,10 +2736,18 @@ var stream_transform_sync = (function (exports) { }; } + if (process.noDeprecation === true) { + return fn; + } + var warned = false; function deprecated() { if (!warned) { - { + if (process.throwDeprecation) { + throw new Error(msg); + } else if (process.traceDeprecation) { + console.trace(msg); + } else { console.error(msg); } warned = true; @@ -2668,7 +2762,7 @@ var stream_transform_sync = (function (exports) { var debugEnviron; function debuglog(set) { if (isUndefined(debugEnviron)) - debugEnviron = ''; + debugEnviron = process.env.NODE_DEBUG || ''; set = set.toUpperCase(); if (!debugs[set]) { if (new RegExp('\\b' + set + '\\b', 'i').test(debugEnviron)) { diff --git a/packages/stream-transform/dist/umd/index.js b/packages/stream-transform/dist/umd/index.js index 65a083a8c..76cae0c45 100644 --- a/packages/stream-transform/dist/umd/index.js +++ b/packages/stream-transform/dist/umd/index.js @@ -705,6 +705,11 @@ ? global$1.TYPED_ARRAY_SUPPORT : true; + /* + * Export kMaxLength after typed array support is determined. + */ + kMaxLength(); + function kMaxLength () { return Buffer.TYPED_ARRAY_SUPPORT ? 0x7fffffff @@ -796,6 +801,8 @@ if (Buffer.TYPED_ARRAY_SUPPORT) { Buffer.prototype.__proto__ = Uint8Array.prototype; Buffer.__proto__ = Uint8Array; + if (typeof Symbol !== 'undefined' && Symbol.species && + Buffer[Symbol.species] === Buffer) ; } function assertSize (size) { @@ -2568,16 +2575,95 @@ Item.prototype.run = function () { this.fun.apply(null, this.array); }; + var title = 'browser'; + var platform = 'browser'; + var browser = true; + var env = {}; + var argv = []; + var version = ''; // empty string to avoid regexp issues + var versions = {}; + var release = {}; + var config = {}; + + function noop() {} + + var on = noop; + var addListener = noop; + var once = noop; + var off = noop; + var removeListener = noop; + var removeAllListeners = noop; + var emit = noop; + + function binding(name) { + throw new Error('process.binding is not supported'); + } + + function cwd () { return '/' } + function chdir (dir) { + throw new Error('process.chdir is not supported'); + }function umask() { return 0; } // from https://github.com/kumavis/browser-process-hrtime/blob/master/index.js var performance = global$1.performance || {}; - performance.now || + var performanceNow = + performance.now || performance.mozNow || performance.msNow || performance.oNow || performance.webkitNow || function(){ return (new Date()).getTime() }; + // generate timestamp or delta + // see http://nodejs.org/api/process.html#process_process_hrtime + function hrtime(previousTimestamp){ + var clocktime = performanceNow.call(performance)*1e-3; + var seconds = Math.floor(clocktime); + var nanoseconds = Math.floor((clocktime%1)*1e9); + if (previousTimestamp) { + seconds = seconds - previousTimestamp[0]; + nanoseconds = nanoseconds - previousTimestamp[1]; + if (nanoseconds<0) { + seconds--; + nanoseconds += 1e9; + } + } + return [seconds,nanoseconds] + } + + var startTime = new Date(); + function uptime() { + var currentTime = new Date(); + var dif = currentTime - startTime; + return dif / 1000; + } + + var process = { + nextTick: nextTick, + title: title, + browser: browser, + env: env, + argv: argv, + version: version, + versions: versions, + on: on, + addListener: addListener, + once: once, + off: off, + removeListener: removeListener, + removeAllListeners: removeAllListeners, + emit: emit, + binding: binding, + cwd: cwd, + chdir: chdir, + umask: umask, + hrtime: hrtime, + platform: platform, + release: release, + config: config, + uptime: uptime + }; + var inherits; if (typeof Object.create === 'function'){ inherits = function inherits(ctor, superCtor) { @@ -2653,10 +2739,18 @@ }; } + if (process.noDeprecation === true) { + return fn; + } + var warned = false; function deprecated() { if (!warned) { - { + if (process.throwDeprecation) { + throw new Error(msg); + } else if (process.traceDeprecation) { + console.trace(msg); + } else { console.error(msg); } warned = true; @@ -2671,7 +2765,7 @@ var debugEnviron; function debuglog(set) { if (isUndefined(debugEnviron)) - debugEnviron = ''; + debugEnviron = process.env.NODE_DEBUG || ''; set = set.toUpperCase(); if (!debugs[set]) { if (new RegExp('\\b' + set + '\\b', 'i').test(debugEnviron)) { diff --git a/packages/stream-transform/dist/umd/sync.js b/packages/stream-transform/dist/umd/sync.js index 88dc02ce7..44b16714d 100644 --- a/packages/stream-transform/dist/umd/sync.js +++ b/packages/stream-transform/dist/umd/sync.js @@ -705,6 +705,11 @@ ? global$1.TYPED_ARRAY_SUPPORT : true; + /* + * Export kMaxLength after typed array support is determined. + */ + kMaxLength(); + function kMaxLength () { return Buffer.TYPED_ARRAY_SUPPORT ? 0x7fffffff @@ -796,6 +801,8 @@ if (Buffer.TYPED_ARRAY_SUPPORT) { Buffer.prototype.__proto__ = Uint8Array.prototype; Buffer.__proto__ = Uint8Array; + if (typeof Symbol !== 'undefined' && Symbol.species && + Buffer[Symbol.species] === Buffer) ; } function assertSize (size) { @@ -2568,16 +2575,95 @@ Item.prototype.run = function () { this.fun.apply(null, this.array); }; + var title = 'browser'; + var platform = 'browser'; + var browser = true; + var env = {}; + var argv = []; + var version = ''; // empty string to avoid regexp issues + var versions = {}; + var release = {}; + var config = {}; + + function noop() {} + + var on = noop; + var addListener = noop; + var once = noop; + var off = noop; + var removeListener = noop; + var removeAllListeners = noop; + var emit = noop; + + function binding(name) { + throw new Error('process.binding is not supported'); + } + + function cwd () { return '/' } + function chdir (dir) { + throw new Error('process.chdir is not supported'); + }function umask() { return 0; } // from https://github.com/kumavis/browser-process-hrtime/blob/master/index.js var performance = global$1.performance || {}; - performance.now || + var performanceNow = + performance.now || performance.mozNow || performance.msNow || performance.oNow || performance.webkitNow || function(){ return (new Date()).getTime() }; + // generate timestamp or delta + // see http://nodejs.org/api/process.html#process_process_hrtime + function hrtime(previousTimestamp){ + var clocktime = performanceNow.call(performance)*1e-3; + var seconds = Math.floor(clocktime); + var nanoseconds = Math.floor((clocktime%1)*1e9); + if (previousTimestamp) { + seconds = seconds - previousTimestamp[0]; + nanoseconds = nanoseconds - previousTimestamp[1]; + if (nanoseconds<0) { + seconds--; + nanoseconds += 1e9; + } + } + return [seconds,nanoseconds] + } + + var startTime = new Date(); + function uptime() { + var currentTime = new Date(); + var dif = currentTime - startTime; + return dif / 1000; + } + + var process = { + nextTick: nextTick, + title: title, + browser: browser, + env: env, + argv: argv, + version: version, + versions: versions, + on: on, + addListener: addListener, + once: once, + off: off, + removeListener: removeListener, + removeAllListeners: removeAllListeners, + emit: emit, + binding: binding, + cwd: cwd, + chdir: chdir, + umask: umask, + hrtime: hrtime, + platform: platform, + release: release, + config: config, + uptime: uptime + }; + var inherits; if (typeof Object.create === 'function'){ inherits = function inherits(ctor, superCtor) { @@ -2653,10 +2739,18 @@ }; } + if (process.noDeprecation === true) { + return fn; + } + var warned = false; function deprecated() { if (!warned) { - { + if (process.throwDeprecation) { + throw new Error(msg); + } else if (process.traceDeprecation) { + console.trace(msg); + } else { console.error(msg); } warned = true; @@ -2671,7 +2765,7 @@ var debugEnviron; function debuglog(set) { if (isUndefined(debugEnviron)) - debugEnviron = ''; + debugEnviron = process.env.NODE_DEBUG || ''; set = set.toUpperCase(); if (!debugs[set]) { if (new RegExp('\\b' + set + '\\b', 'i').test(debugEnviron)) { diff --git a/packages/stream-transform/package.json b/packages/stream-transform/package.json index 20d4d77c2..ca2bc5848 100644 --- a/packages/stream-transform/package.json +++ b/packages/stream-transform/package.json @@ -10,22 +10,22 @@ ], "author": "David Worms (https://www.adaltas.com)", "devDependencies": { - "@rollup/plugin-eslint": "^8.0.1", - "@rollup/plugin-node-resolve": "^13.1.3", - "@types/mocha": "^9.1.0", - "@types/node": "^17.0.18", - "coffeescript": "~2.6.1", + "@rollup/plugin-eslint": "^8.0.2", + "@rollup/plugin-node-resolve": "^13.3.0", + "@types/mocha": "^9.1.1", + "@types/node": "^17.0.35", + "coffeescript": "~2.7.0", "csv-generate": "^4.0.4", "each": "^1.2.2", - "eslint": "^8.9.0", - "mocha": "~9.2.0", + "eslint": "^8.16.0", + "mocha": "~10.0.0", "pad": "~3.2.0", - "rollup": "^2.67.2", + "rollup": "^2.74.1", "rollup-plugin-node-builtins": "^2.1.2", "rollup-plugin-node-globals": "^1.4.0", "should": "~13.2.3", - "ts-node": "^10.5.0", - "typescript": "^4.5.5" + "ts-node": "^10.8.0", + "typescript": "^4.6.4" }, "exports": { ".": { From 28d60660de1c886e51e9cc16771f17fc4257a304 Mon Sep 17 00:00:00 2001 From: David Worms Date: Tue, 24 May 2022 09:47:54 +0200 Subject: [PATCH 15/19] fix(csv-demo-eslint): private package --- demo/eslint/package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/demo/eslint/package.json b/demo/eslint/package.json index ad5f58aeb..f4c5d417b 100644 --- a/demo/eslint/package.json +++ b/demo/eslint/package.json @@ -3,6 +3,7 @@ "version": "0.0.1", "description": "", "main": "index.js", + "private": true, "scripts": { "lint": "eslint ./", "test": "echo \"Not ready, check output of npm run lint\"" From 1cb60b062ba5ccf1c95690c2e305173071be816e Mon Sep 17 00:00:00 2001 From: David Worms Date: Tue, 24 May 2022 10:00:32 +0200 Subject: [PATCH 16/19] build: conventional commit property in lerna conf --- lerna.json | 1 + package.json | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index d107e4061..ea09e6f21 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,7 @@ { "command": { "version": { + "conventionalCommits": true, "message": "chore(release): publish" } }, diff --git a/package.json b/package.json index a8300a671..08729ce78 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "publish": "lerna publish from-git --yes", "test": "lerna run test", "test:legacy": "lerna run test:legacy", - "version": "lerna version --conventional-commits" + "version": "lerna version" }, "workspaces": { "packages": [ From ddb9fab7221a4bc1ab05f420b444fb45ace94f83 Mon Sep 17 00:00:00 2001 From: David Worms Date: Tue, 24 May 2022 10:18:04 +0200 Subject: [PATCH 17/19] build: changelog convertional commit preset --- lerna.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lerna.json b/lerna.json index ea09e6f21..1204e21da 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,10 @@ { "command": { "version": { + "changelogPreset": { + "name": "conventionalcommits", + "issueUrlFormat": "{{host}}/{{owner}}/{{repository}}/issues/{{id}}" + }, "conventionalCommits": true, "message": "chore(release): publish" } From ea23426cdeca967898be0192ab8a99ae0eed5b2f Mon Sep 17 00:00:00 2001 From: David Worms Date: Tue, 24 May 2022 10:24:07 +0200 Subject: [PATCH 18/19] ci: test node 14, 16 and 18 --- .github/workflows/nodejs.yml | 4 ++-- packages/csv-generate/package.json | 2 +- packages/csv-parse/package.json | 2 +- packages/csv-stringify/package.json | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/nodejs.yml b/.github/workflows/nodejs.yml index 86b6076bf..99d419c6f 100644 --- a/.github/workflows/nodejs.yml +++ b/.github/workflows/nodejs.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - node-version: [16.x] + node-version: [16.x, 18.x] steps: - uses: actions/checkout@v2 - name: Use Node.js ${{ matrix.node-version }} @@ -25,7 +25,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - node-version: [12.x, 14.x] + node-version: [14.x] steps: - uses: actions/checkout@v2 - name: Use Node.js ${{ matrix.node-version }} diff --git a/packages/csv-generate/package.json b/packages/csv-generate/package.json index 1f3d2acfa..ded6a3460 100644 --- a/packages/csv-generate/package.json +++ b/packages/csv-generate/package.json @@ -77,7 +77,7 @@ "preversion": "npm run build && git add dist", "pretest": "npm run build", "test": "mocha 'test/**/*.{coffee,ts}'", - "test:legacy": "mocha --loader=./test/loaders/legacy/all.js 'test/**/*.{coffee,ts}' --ignore test/api.web_stream.coffee" + "test:legacy": "mocha --ignore test/api.web_stream.coffee --loader=./test/loaders/legacy/all.js 'test/**/*.{coffee,ts}'" }, "type": "module", "types": "dist/esm/index.d.ts", diff --git a/packages/csv-parse/package.json b/packages/csv-parse/package.json index 15e0994cd..59abf7b6a 100644 --- a/packages/csv-parse/package.json +++ b/packages/csv-parse/package.json @@ -96,7 +96,7 @@ "preversion": "npm run build && git add dist", "pretest": "npm run build", "test": "mocha 'test/**/*.{coffee,ts}'", - "test:legacy": "mocha --loader=./test/loaders/legacy/all.js 'test/**/*.{coffee,ts}' --ignore test/api.web_stream.coffee" + "test:legacy": "mocha --ignore test/api.web_stream.coffee --loader=./test/loaders/legacy/all.js 'test/**/*.{coffee,ts}'" }, "type": "module", "types": "dist/esm/index.d.ts", diff --git a/packages/csv-stringify/package.json b/packages/csv-stringify/package.json index 30783560f..3c3b89f6e 100644 --- a/packages/csv-stringify/package.json +++ b/packages/csv-stringify/package.json @@ -73,7 +73,7 @@ "preversion": "npm run build && git add dist", "pretest": "npm run build", "test": "mocha 'test/**/*.{coffee,ts}'", - "test:legacy": "mocha --loader=./test/loaders/legacy/all.js 'test/**/*.{coffee,ts}' --ignore test/api.web_stream.coffee" + "test:legacy": "mocha --ignore test/api.web_stream.coffee --loader=./test/loaders/legacy/all.js 'test/**/*.{coffee,ts}'" }, "type": "module", "types": "dist/esm/index.d.ts", From 59cf7a4333c08020a029fa6922483f058bec04ab Mon Sep 17 00:00:00 2001 From: David Worms Date: Tue, 24 May 2022 10:41:46 +0200 Subject: [PATCH 19/19] chore(release): publish - csv-demo-browser@0.1.2 - csv-demo-cjs@0.1.2 - csv-demo-eslint@0.1.0 - csv-demo-esm@0.0.3 - csv-issues-cjs@0.1.0 - csv-issues-esm@0.0.2 - csv-demo-webpack-ts@0.1.1 - csv-demo-webpack@0.1.3 - csv-generate@4.1.0 - csv-parse@5.1.0 - csv-stringify@6.1.0 - csv@6.1.0 - stream-transform@3.1.0 --- demo/browser/CHANGELOG.md | 8 ++++++++ demo/browser/package.json | 2 +- demo/cjs/CHANGELOG.md | 8 ++++++++ demo/cjs/package.json | 2 +- demo/eslint/CHANGELOG.md | 16 ++++++++++++++++ demo/eslint/package.json | 4 ++-- demo/esm/CHANGELOG.md | 9 +++++++++ demo/esm/package.json | 6 +++--- demo/issues-cjs/CHANGELOG.md | 9 +++++++++ demo/issues-cjs/package.json | 2 +- demo/issues-esm/CHANGELOG.md | 8 ++++++++ demo/issues-esm/package.json | 2 +- demo/webpack-ts/CHANGELOG.md | 8 ++++++++ demo/webpack-ts/package.json | 2 +- demo/webpack/CHANGELOG.md | 8 ++++++++ demo/webpack/package.json | 2 +- packages/csv-generate/CHANGELOG.md | 14 ++++++++++++++ packages/csv-generate/package.json | 2 +- packages/csv-parse/CHANGELOG.md | 9 +++++++++ packages/csv-parse/package.json | 6 +++--- packages/csv-stringify/CHANGELOG.md | 14 ++++++++++++++ packages/csv-stringify/package.json | 4 ++-- packages/csv/CHANGELOG.md | 9 +++++++++ packages/csv/package.json | 10 +++++----- packages/stream-transform/CHANGELOG.md | 9 +++++++++ packages/stream-transform/package.json | 4 ++-- 26 files changed, 153 insertions(+), 24 deletions(-) create mode 100644 demo/eslint/CHANGELOG.md diff --git a/demo/browser/CHANGELOG.md b/demo/browser/CHANGELOG.md index 1503d5e98..7c6cfa47a 100644 --- a/demo/browser/CHANGELOG.md +++ b/demo/browser/CHANGELOG.md @@ -3,6 +3,14 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +### [0.1.2](https://github.com/adaltas/node-csv/compare/csv-demo-browser@0.1.1...csv-demo-browser@0.1.2) (2022-05-24) + +**Note:** Version bump only for package csv-demo-browser + + + + + ## [0.1.1](https://github.com/adaltas/node-csv/compare/csv-demo-browser@0.1.0...csv-demo-browser@0.1.1) (2021-12-29) **Note:** Version bump only for package csv-demo-browser diff --git a/demo/browser/package.json b/demo/browser/package.json index 68bdcc54d..5b365c750 100644 --- a/demo/browser/package.json +++ b/demo/browser/package.json @@ -1,6 +1,6 @@ { "name": "csv-demo-browser", - "version": "0.1.1", + "version": "0.1.2", "main": "index.js", "license": "MIT", "type": "module", diff --git a/demo/cjs/CHANGELOG.md b/demo/cjs/CHANGELOG.md index b7b0daa0b..7f88643a8 100644 --- a/demo/cjs/CHANGELOG.md +++ b/demo/cjs/CHANGELOG.md @@ -3,6 +3,14 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +### [0.1.2](https://github.com/adaltas/node-csv/compare/csv-demo-cjs@0.1.1...csv-demo-cjs@0.1.2) (2022-05-24) + +**Note:** Version bump only for package csv-demo-cjs + + + + + ## [0.1.1](https://github.com/adaltas/node-csv/compare/csv-demo-cjs@0.1.0...csv-demo-cjs@0.1.1) (2021-12-29) **Note:** Version bump only for package csv-demo-cjs diff --git a/demo/cjs/package.json b/demo/cjs/package.json index 50bf40dea..131e39ab3 100644 --- a/demo/cjs/package.json +++ b/demo/cjs/package.json @@ -1,6 +1,6 @@ { "name": "csv-demo-cjs", - "version": "0.1.1", + "version": "0.1.2", "main": "index.js", "license": "MIT", "type": "commonjs", diff --git a/demo/eslint/CHANGELOG.md b/demo/eslint/CHANGELOG.md new file mode 100644 index 000000000..d5a3cc720 --- /dev/null +++ b/demo/eslint/CHANGELOG.md @@ -0,0 +1,16 @@ +# Change Log + +All notable changes to this project will be documented in this file. +See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. + +## 0.1.0 (2022-05-24) + + +### Features + +* wg stream api ([8a5eb7d](https://github.com/adaltas/node-csv/commit/8a5eb7dfd31b22217db4fbbc832d707221850785)) + + +### Bug Fixes + +* **csv-demo-eslint:** private package ([28d6066](https://github.com/adaltas/node-csv/commit/28d60660de1c886e51e9cc16771f17fc4257a304)) diff --git a/demo/eslint/package.json b/demo/eslint/package.json index f4c5d417b..682e83f57 100644 --- a/demo/eslint/package.json +++ b/demo/eslint/package.json @@ -1,6 +1,6 @@ { "name": "csv-demo-eslint", - "version": "0.0.1", + "version": "0.1.0", "description": "", "main": "index.js", "private": true, @@ -10,7 +10,7 @@ }, "license": "MIT", "dependencies": { - "csv-stringify": "^6.0.5" + "csv-stringify": "^6.1.0" }, "devDependencies": { "eslint": "^8.16.0", diff --git a/demo/esm/CHANGELOG.md b/demo/esm/CHANGELOG.md index d3acf82a3..09d111bd8 100644 --- a/demo/esm/CHANGELOG.md +++ b/demo/esm/CHANGELOG.md @@ -3,6 +3,15 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +### [0.0.3](https://github.com/adaltas/node-csv/compare/csv-demo-esm@0.0.2...csv-demo-esm@0.0.3) (2022-05-24) + + +### Bug Fixes + +* **csv-demo-esm:** csv dependencies ([64afead](https://github.com/adaltas/node-csv/commit/64afead8dc41b9d379c9761ddb70d6a29251b4e2)) + + + ## [0.0.2](https://github.com/adaltas/node-csv/compare/csv-demo-esm@0.0.1...csv-demo-esm@0.0.2) (2021-12-29) **Note:** Version bump only for package csv-demo-esm diff --git a/demo/esm/package.json b/demo/esm/package.json index 1a44334ae..0f4de50ad 100644 --- a/demo/esm/package.json +++ b/demo/esm/package.json @@ -1,13 +1,13 @@ { "name": "csv-demo-esm", - "version": "0.0.2", + "version": "0.0.3", "main": "index.js", "license": "MIT", "type": "module", "private": true, "dependencies": { - "csv": "^6.0.5", - "csv-parse": "^5.0.4" + "csv": "^6.1.0", + "csv-parse": "^5.1.0" }, "devDependencies": { "coffeescript": "^2.7.0", diff --git a/demo/issues-cjs/CHANGELOG.md b/demo/issues-cjs/CHANGELOG.md index a8200653a..873c9ad62 100644 --- a/demo/issues-cjs/CHANGELOG.md +++ b/demo/issues-cjs/CHANGELOG.md @@ -3,6 +3,15 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [0.1.0](https://github.com/adaltas/node-csv/compare/csv-issues-cjs@0.0.2...csv-issues-cjs@0.1.0) (2022-05-24) + + +### Features + +* **csv-issues-cjs:** 330 sample code ([3d85a41](https://github.com/adaltas/node-csv/commit/3d85a411007416f3cb750ca6b427f55c0331a8b8)) + + + ## [0.0.2](https://github.com/adaltas/node-csv/compare/csv-issues-cjs@0.0.1...csv-issues-cjs@0.0.2) (2021-11-19) diff --git a/demo/issues-cjs/package.json b/demo/issues-cjs/package.json index 777750ef0..11aeb1b1d 100644 --- a/demo/issues-cjs/package.json +++ b/demo/issues-cjs/package.json @@ -1,6 +1,6 @@ { "name": "csv-issues-cjs", - "version": "0.0.2", + "version": "0.1.0", "main": "index.js", "license": "MIT", "private": true, diff --git a/demo/issues-esm/CHANGELOG.md b/demo/issues-esm/CHANGELOG.md index 65973ccb1..4bc7a0025 100644 --- a/demo/issues-esm/CHANGELOG.md +++ b/demo/issues-esm/CHANGELOG.md @@ -3,6 +3,14 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +### [0.0.2](https://github.com/adaltas/node-csv/compare/csv-issues-esm@0.0.1...csv-issues-esm@0.0.2) (2022-05-24) + +**Note:** Version bump only for package csv-issues-esm + + + + + ## 0.0.1 (2021-11-19) **Note:** Version bump only for package csv-issues-esm diff --git a/demo/issues-esm/package.json b/demo/issues-esm/package.json index 7f667d9ce..815b09721 100644 --- a/demo/issues-esm/package.json +++ b/demo/issues-esm/package.json @@ -1,6 +1,6 @@ { "name": "csv-issues-esm", - "version": "0.0.1", + "version": "0.0.2", "main": "index.js", "license": "MIT", "type": "module", diff --git a/demo/webpack-ts/CHANGELOG.md b/demo/webpack-ts/CHANGELOG.md index 7215358f6..539cb9701 100644 --- a/demo/webpack-ts/CHANGELOG.md +++ b/demo/webpack-ts/CHANGELOG.md @@ -3,6 +3,14 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +### [0.1.1](https://github.com/adaltas/node-csv/compare/csv-demo-webpack-ts@0.1.0...csv-demo-webpack-ts@0.1.1) (2022-05-24) + +**Note:** Version bump only for package csv-demo-webpack-ts + + + + + # 0.1.0 (2021-12-29) diff --git a/demo/webpack-ts/package.json b/demo/webpack-ts/package.json index 182e1172b..8cee5439f 100644 --- a/demo/webpack-ts/package.json +++ b/demo/webpack-ts/package.json @@ -1,6 +1,6 @@ { "name": "csv-demo-webpack-ts", - "version": "0.1.0", + "version": "0.1.1", "description": "", "private": true, "scripts": { diff --git a/demo/webpack/CHANGELOG.md b/demo/webpack/CHANGELOG.md index 7c8b87146..ae4f8937e 100644 --- a/demo/webpack/CHANGELOG.md +++ b/demo/webpack/CHANGELOG.md @@ -3,6 +3,14 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +### [0.1.3](https://github.com/adaltas/node-csv/compare/csv-demo-webpack@0.1.2...csv-demo-webpack@0.1.3) (2022-05-24) + +**Note:** Version bump only for package csv-demo-webpack + + + + + ## [0.1.2](https://github.com/adaltas/node-csv/compare/csv-demo-webpack@0.1.1...csv-demo-webpack@0.1.2) (2021-12-29) diff --git a/demo/webpack/package.json b/demo/webpack/package.json index 106b7052b..fc1557ef3 100644 --- a/demo/webpack/package.json +++ b/demo/webpack/package.json @@ -1,6 +1,6 @@ { "name": "csv-demo-webpack", - "version": "0.1.2", + "version": "0.1.3", "description": "", "private": true, "scripts": { diff --git a/packages/csv-generate/CHANGELOG.md b/packages/csv-generate/CHANGELOG.md index ce2b5e5b7..d9de82a77 100644 --- a/packages/csv-generate/CHANGELOG.md +++ b/packages/csv-generate/CHANGELOG.md @@ -3,6 +3,20 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [4.1.0](https://github.com/adaltas/node-csv/compare/csv-generate@4.0.4...csv-generate@4.1.0) (2022-05-24) + + +### Features + +* wg stream api ([8a5eb7d](https://github.com/adaltas/node-csv/commit/8a5eb7dfd31b22217db4fbbc832d707221850785)) + + +### Bug Fixes + +* **csv-generate:** catch invalid value error ([f031542](https://github.com/adaltas/node-csv/commit/f0315423ba576551f2bd08f3e1c3bc85e9003926)) + + + ## [4.0.4](https://github.com/adaltas/node-csv/compare/csv-generate@4.0.3...csv-generate@4.0.4) (2021-12-29) diff --git a/packages/csv-generate/package.json b/packages/csv-generate/package.json index ded6a3460..81766e909 100644 --- a/packages/csv-generate/package.json +++ b/packages/csv-generate/package.json @@ -1,5 +1,5 @@ { - "version": "4.0.4", + "version": "4.1.0", "name": "csv-generate", "description": "CSV and object generation implementing the Node.js `stream.Readable` API", "keywords": [ diff --git a/packages/csv-parse/CHANGELOG.md b/packages/csv-parse/CHANGELOG.md index 6559deb3f..1a44a2e06 100644 --- a/packages/csv-parse/CHANGELOG.md +++ b/packages/csv-parse/CHANGELOG.md @@ -3,6 +3,15 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [5.1.0](https://github.com/adaltas/node-csv/compare/csv-parse@5.0.4...csv-parse@5.1.0) (2022-05-24) + + +### Features + +* wg stream api ([8a5eb7d](https://github.com/adaltas/node-csv/commit/8a5eb7dfd31b22217db4fbbc832d707221850785)) + + + ## [5.0.4](https://github.com/adaltas/node-csv/compare/csv-parse@5.0.3...csv-parse@5.0.4) (2021-12-29) diff --git a/packages/csv-parse/package.json b/packages/csv-parse/package.json index 59abf7b6a..d176b3cd5 100644 --- a/packages/csv-parse/package.json +++ b/packages/csv-parse/package.json @@ -1,5 +1,5 @@ { - "version": "5.0.4", + "version": "5.1.0", "name": "csv-parse", "description": "CSV parsing implementing the Node.js `stream.Transform` API", "keywords": [ @@ -48,7 +48,7 @@ "@types/node": "^17.0.35", "coffeelint": "^2.1.0", "coffeescript": "^2.7.0", - "csv-generate": "^4.0.4", + "csv-generate": "^4.1.0", "csv-spectrum": "^1.0.0", "each": "^1.2.2", "eslint": "^8.16.0", @@ -58,7 +58,7 @@ "rollup-plugin-node-builtins": "^2.1.2", "rollup-plugin-node-globals": "^1.4.0", "should": "^13.2.3", - "stream-transform": "^3.0.4", + "stream-transform": "^3.1.0", "ts-node": "^10.8.0", "typescript": "^4.6.4" }, diff --git a/packages/csv-stringify/CHANGELOG.md b/packages/csv-stringify/CHANGELOG.md index 663e1e8b5..919344fb5 100644 --- a/packages/csv-stringify/CHANGELOG.md +++ b/packages/csv-stringify/CHANGELOG.md @@ -3,6 +3,20 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [6.1.0](https://github.com/adaltas/node-csv/compare/csv-stringify@6.0.5...csv-stringify@6.1.0) (2022-05-24) + + +### Features + +* wg stream api ([8a5eb7d](https://github.com/adaltas/node-csv/commit/8a5eb7dfd31b22217db4fbbc832d707221850785)) + + +### Bug Fixes + +* **csv-stringify:** update TS types to allow cast to return an object ([#339](https://github.com/adaltas/node-csv/issues/339)) ([60efa78](https://github.com/adaltas/node-csv/commit/60efa7862ed43bd2fd19d1f027a1809e9df6a67e)) + + + ## [6.0.5](https://github.com/adaltas/node-csv/compare/csv-stringify@6.0.4...csv-stringify@6.0.5) (2021-12-29) diff --git a/packages/csv-stringify/package.json b/packages/csv-stringify/package.json index 3c3b89f6e..96982a4f3 100644 --- a/packages/csv-stringify/package.json +++ b/packages/csv-stringify/package.json @@ -1,5 +1,5 @@ { - "version": "6.0.5", + "version": "6.1.0", "name": "csv-stringify", "description": "CSV stringifier implementing the Node.js `stream.Transform` API", "keywords": [ @@ -15,7 +15,7 @@ "@types/node": "^17.0.35", "@types/should": "^13.0.0", "coffeescript": "~2.7.0", - "csv-generate": "^4.0.4", + "csv-generate": "^4.1.0", "each": "^1.2.2", "eslint": "^8.16.0", "express": "^4.18.1", diff --git a/packages/csv/CHANGELOG.md b/packages/csv/CHANGELOG.md index 42968a906..af50df563 100644 --- a/packages/csv/CHANGELOG.md +++ b/packages/csv/CHANGELOG.md @@ -3,6 +3,15 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [6.1.0](https://github.com/adaltas/node-csv/compare/csv@6.0.5...csv@6.1.0) (2022-05-24) + + +### Features + +* wg stream api ([8a5eb7d](https://github.com/adaltas/node-csv/commit/8a5eb7dfd31b22217db4fbbc832d707221850785)) + + + ## [6.0.5](https://github.com/adaltas/node-csv/compare/csv@6.0.4...csv@6.0.5) (2021-12-29) diff --git a/packages/csv/package.json b/packages/csv/package.json index dd449752f..84c32ed67 100644 --- a/packages/csv/package.json +++ b/packages/csv/package.json @@ -1,6 +1,6 @@ { "name": "csv", - "version": "6.0.5", + "version": "6.1.0", "description": "A mature CSV toolset with simple api, full of options and tested against large datasets.", "keywords": [ "node", @@ -21,10 +21,10 @@ "David Worms (https://www.adaltas.com)" ], "dependencies": { - "csv-generate": "^4.0.4", - "csv-parse": "^5.0.4", - "csv-stringify": "^6.0.5", - "stream-transform": "^3.0.4" + "csv-generate": "^4.1.0", + "csv-parse": "^5.1.0", + "csv-stringify": "^6.1.0", + "stream-transform": "^3.1.0" }, "devDependencies": { "@rollup/plugin-eslint": "^8.0.2", diff --git a/packages/stream-transform/CHANGELOG.md b/packages/stream-transform/CHANGELOG.md index c5ba01868..c1c9bb239 100644 --- a/packages/stream-transform/CHANGELOG.md +++ b/packages/stream-transform/CHANGELOG.md @@ -3,6 +3,15 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [3.1.0](https://github.com/adaltas/node-csv/compare/stream-transform@3.0.4...stream-transform@3.1.0) (2022-05-24) + + +### Features + +* wg stream api ([8a5eb7d](https://github.com/adaltas/node-csv/commit/8a5eb7dfd31b22217db4fbbc832d707221850785)) + + + ## [3.0.4](https://github.com/adaltas/node-csv/compare/stream-transform@3.0.3...stream-transform@3.0.4) (2021-12-29) diff --git a/packages/stream-transform/package.json b/packages/stream-transform/package.json index ca2bc5848..3d85f9fb5 100644 --- a/packages/stream-transform/package.json +++ b/packages/stream-transform/package.json @@ -1,5 +1,5 @@ { - "version": "3.0.4", + "version": "3.1.0", "name": "stream-transform", "description": "Object transformations implementing the Node.js `stream.Transform` API", "keywords": [ @@ -15,7 +15,7 @@ "@types/mocha": "^9.1.1", "@types/node": "^17.0.35", "coffeescript": "~2.7.0", - "csv-generate": "^4.0.4", + "csv-generate": "^4.1.0", "each": "^1.2.2", "eslint": "^8.16.0", "mocha": "~10.0.0",