diff --git a/notebook/Zeppelin Tutorial/Matplotlib (Python | PySpark)_2C2AUG798.zpln b/notebook/Zeppelin Tutorial/Matplotlib (Python PySpark)_2C2AUG798.zpln similarity index 100% rename from notebook/Zeppelin Tutorial/Matplotlib (Python | PySpark)_2C2AUG798.zpln rename to notebook/Zeppelin Tutorial/Matplotlib (Python PySpark)_2C2AUG798.zpln diff --git a/pom.xml b/pom.xml index 786c337ef5e..fc6844a85f3 100644 --- a/pom.xml +++ b/pom.xml @@ -52,6 +52,7 @@ 2013 + zeppelin-mongodb-interpreter zeppelin-interpreter-parent zeppelin-interpreter zeppelin-interpreter-api @@ -100,7 +101,7 @@ 1.12.5 - v8.9.3 + v8.11.3 5.5.1 1.4 diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java index b76634a104d..699da475374 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java @@ -17,14 +17,6 @@ package org.apache.zeppelin.conf; -import java.io.File; -import java.net.URL; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.function.Predicate; - import org.apache.commons.configuration.ConfigurationException; import org.apache.commons.configuration.XMLConfiguration; import org.apache.commons.configuration.tree.ConfigurationNode; @@ -33,6 +25,14 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; +import java.net.URL; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Predicate; + /** * Zeppelin configuration. * @@ -201,11 +201,11 @@ public int getInt(ConfVars c) { public int getInt(String envName, String propertyName, int defaultValue) { if (System.getenv(envName) != null) { - return Integer.parseInt(System.getenv(envName)); + return Integer.parseInt(getValue(System.getenv(envName))); } if (System.getProperty(propertyName) != null) { - return Integer.parseInt(System.getProperty(propertyName)); + return Integer.parseInt(getValue(System.getProperty(propertyName))); } return getIntValue(propertyName, defaultValue); } @@ -216,15 +216,22 @@ public long getLong(ConfVars c) { public long getLong(String envName, String propertyName, long defaultValue) { if (System.getenv(envName) != null) { - return Long.parseLong(System.getenv(envName)); + return Long.parseLong(getValue(System.getenv(envName))); } if (System.getProperty(propertyName) != null) { - return Long.parseLong(System.getProperty(propertyName)); + return Long.parseLong(getValue(System.getProperty(propertyName))); } return getLongValue(propertyName, defaultValue); } + private String getValue(String input) { + if (input.contains(":")) { + return input.substring(input.lastIndexOf(":") + 1); + } else + return input; + } + public float getFloat(ConfVars c) { return getFloat(c.name(), c.getVarName(), c.getFloatValue()); } diff --git a/zeppelin-mongodb-interpreter/.gitignore b/zeppelin-mongodb-interpreter/.gitignore new file mode 100644 index 00000000000..4aa6e0713b1 --- /dev/null +++ b/zeppelin-mongodb-interpreter/.gitignore @@ -0,0 +1,12 @@ +target/ +pom.xml.tag +pom.xml.releaseBackup +pom.xml.versionsBackup +pom.xml.next +release.properties +dependency-reduced-pom.xml +buildNumber.properties +.mvn/timing.properties +.classpath +.project +.settings/ diff --git a/zeppelin-mongodb-interpreter/LICENSE b/zeppelin-mongodb-interpreter/LICENSE new file mode 100644 index 00000000000..8dada3edaf5 --- /dev/null +++ b/zeppelin-mongodb-interpreter/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/zeppelin-mongodb-interpreter/README.md b/zeppelin-mongodb-interpreter/README.md new file mode 100644 index 00000000000..9863ec782b3 --- /dev/null +++ b/zeppelin-mongodb-interpreter/README.md @@ -0,0 +1,100 @@ +# zeppelin-mongodb-interpreter +MongoDB interpreter for Apache Zeppelin. + +__Supported versions of MongoDB: >= 3.0__ + + +> If you are interested, there is a Docker image for Zeppelin with MongoDB interpreter: https://hub.docker.com/r/cthiebault/zeppelin-mongodb/ + +## Build + +Requirement: Zeppelin must be in your local repo. + +```sh +mvn clean package +``` + +## Download + +If you cannot build the jar, you can download it in the [release page](https://github.com/bbonnin/zeppelin-mongodb-interpreter/releases) + +## Deployment + +* Update `$ZEPPELIN_HOME/conf/zeppeln-site.xml` +```xml + + zeppelin.interpreters + ...,org.apache.zeppelin.mongodb.MongoDbInterpreter + +``` +* Create `$ZEPPELIN_HOME/interpreter/mongodb` +* Copy interpreter jar in `$ZEPPELIN_HOME/interpreter/mongodb` + + +> In some cases, mongodb is not visible in the list of the available interpreters. In this case, after the previous steps, you can create a new interpreter and in the interpreter group selection, you should be able to select mongodb. + +![Create](docs/zeppelin-mongo-interpreter-install.png) + + + +## Configuration + + + + + + + + + + + + +
ParameterDefault valueDescription
mongo.shell.pathmongoMongo shell path
mongo.shell.command.timeout60000Mongo command timeout
mongo.shell.command.table.limit1000Limit of documents displayed in a table
mongo.server.databasetestMongDB database name
mongo.server.hostlocalhostHost of the MongDB server
mongo.server.port27017Port of the MongDB server
mongo.server.usernameUsername for authentication
mongo.server.passwordPassword for authentication
mongo.server.authentdatabaseDatabase used for authentication
+ +## How to use + +In Zeppelin, use `%mongodb` in a paragraph. +After that, you can type the same Javascript code you use when you write scripts for the Mongo shell. +For more information, please consult: https://docs.mongodb.com/manual/tutorial/write-scripts-for-the-mongo-shell/ + +There are several functions that have been added to help you in Zeppelin: +* printTable(cursor, fields, flattenArray): to print a table (i.e. it uses `%table`). Arguments: + * cursor: a DBQuery or DBCommandCursor instance + * fields: an array of field names to put in the table (can be null) + * flattenArray: if true, the arrays in the documents will also be flatten (false by default) +* DBQuery.prototype.table: to print a table (it invokes the previous function) +* DBCommandCursor.prototype.table: same as above + +Examples: +```javascript +%mongodb + +// Display a table +db.zipcodes.find({ "city":"CHICAGO", "state": "IL" }).table() +``` + +```javascript +%mongodb + +var states = db.zipcodes.aggregate( [ + { $group: { _id: "$state", totalPop: { $sum: "$pop" } } }, + { $match: { totalPop: { $lt: 1000*1000 } } }, + { $sort: { totalPop: 1 } } +] ) + +// Build a 'table' +print("%table state\ttotalPop") +states.forEach(state => { print(state._id + "\t" + state.totalPop) }) +``` + + +## Examples + +* Configuration: +![Configuration](docs/zeppelin-mongo-config.png) + +* Queries (these examples come from: https://docs.mongodb.com/manual/tutorial/aggregation-zip-code-data-set/) +![Examples](docs/zeppelin-mongo-examples.png) + + diff --git a/zeppelin-mongodb-interpreter/docs/zeppelin-mongo-config.png b/zeppelin-mongodb-interpreter/docs/zeppelin-mongo-config.png new file mode 100644 index 00000000000..328b4ceaade Binary files /dev/null and b/zeppelin-mongodb-interpreter/docs/zeppelin-mongo-config.png differ diff --git a/zeppelin-mongodb-interpreter/docs/zeppelin-mongo-examples.png b/zeppelin-mongodb-interpreter/docs/zeppelin-mongo-examples.png new file mode 100644 index 00000000000..0e111c9b23b Binary files /dev/null and b/zeppelin-mongodb-interpreter/docs/zeppelin-mongo-examples.png differ diff --git a/zeppelin-mongodb-interpreter/docs/zeppelin-mongo-interpreter-install.png b/zeppelin-mongodb-interpreter/docs/zeppelin-mongo-interpreter-install.png new file mode 100644 index 00000000000..c78bba690b2 Binary files /dev/null and b/zeppelin-mongodb-interpreter/docs/zeppelin-mongo-interpreter-install.png differ diff --git a/zeppelin-mongodb-interpreter/docs/zeppelin-mongodb-interpreter-monitor.png b/zeppelin-mongodb-interpreter/docs/zeppelin-mongodb-interpreter-monitor.png new file mode 100644 index 00000000000..68cd0dbaba8 Binary files /dev/null and b/zeppelin-mongodb-interpreter/docs/zeppelin-mongodb-interpreter-monitor.png differ diff --git a/zeppelin-mongodb-interpreter/docs/zeppelin-mongodb-interpreter.png b/zeppelin-mongodb-interpreter/docs/zeppelin-mongodb-interpreter.png new file mode 100644 index 00000000000..803d794c030 Binary files /dev/null and b/zeppelin-mongodb-interpreter/docs/zeppelin-mongodb-interpreter.png differ diff --git a/zeppelin-mongodb-interpreter/pom.xml b/zeppelin-mongodb-interpreter/pom.xml new file mode 100644 index 00000000000..d76f96891c8 --- /dev/null +++ b/zeppelin-mongodb-interpreter/pom.xml @@ -0,0 +1,71 @@ + + 4.0.0 + + zeppelin-interpreter-parent + org.apache.zeppelin + 0.9.0-SNAPSHOT + ../zeppelin-interpreter-parent + + + org.apache.zeppelin + zeppelin-mongodb + jar + 0.9.0-SNAPSHOT + Zeppelin: MongoDB interpreter + http://www.apache.org + + + UTF-8 + ${project.version} + 4.11 + mongo + + + + + ${project.groupId} + zeppelin-interpreter + ${project.version} + provided + + + + junit + junit + ${junit.version} + test + + + + + + + maven-compiler-plugin + 3.3 + + 1.7 + 1.7 + + + + maven-assembly-plugin + 2.6 + + + jar-with-dependencies + + + + + make-assembly + package + + single + + + + + + + diff --git a/zeppelin-mongodb-interpreter/src/main/java/org/apache/zeppelin/mongodb/MongoDbInterpreter.java b/zeppelin-mongodb-interpreter/src/main/java/org/apache/zeppelin/mongodb/MongoDbInterpreter.java new file mode 100644 index 00000000000..d55ebea13d1 --- /dev/null +++ b/zeppelin-mongodb-interpreter/src/main/java/org/apache/zeppelin/mongodb/MongoDbInterpreter.java @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.mongodb; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; +import java.util.Scanner; + +import org.apache.commons.exec.CommandLine; +import org.apache.commons.exec.DefaultExecutor; +import org.apache.commons.exec.ExecuteException; +import org.apache.commons.exec.ExecuteWatchdog; +import org.apache.commons.exec.Executor; +import org.apache.commons.exec.PumpStreamHandler; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterResult; +import org.apache.zeppelin.interpreter.InterpreterResult.Code; +import org.apache.zeppelin.scheduler.Scheduler; +import org.apache.zeppelin.scheduler.SchedulerFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * MongoDB interpreter. It uses the mongo shell to interpret the commands. + * + */ +public class MongoDbInterpreter extends Interpreter { + + private static final Logger LOGGER = LoggerFactory.getLogger(MongoDbInterpreter.class); + + private static final String SHELL_EXTENSION = + new Scanner(MongoDbInterpreter.class.getResourceAsStream("/shell_extension.js"), "UTF-8") + .useDelimiter("\\A").next(); + + private long commandTimeout = 60000; + + private String dbAddress; + + private Map runningProcesses = new HashMap<>(); + + + public MongoDbInterpreter(Properties property) { + super(property); + } + + @Override + public void open() { + commandTimeout = Long.parseLong(getProperty("mongo.shell.command.timeout")); + dbAddress = getProperty("mongo.server.host") + ":" + + getProperty("mongo.server.port") + "/" + + getProperty("mongo.server.database"); + } + + @Override + public void close() { + //Nothing to do + } + + @Override + public FormType getFormType() { + return FormType.SIMPLE; + } + + @Override + public InterpreterResult interpret(String script, InterpreterContext context) { + LOGGER.debug("Run MongoDB script: {}", script); + + if (StringUtils.isEmpty(script)) { + return new InterpreterResult(Code.SUCCESS); + } + + // Write script in a temporary file + // The script is enriched with extensions + final File scriptFile = new File(getScriptFileName(context.getParagraphId())); + try { + FileUtils.write(scriptFile, + SHELL_EXTENSION + .replace("__ZEPPELIN_TABLE_LIMIT__", getProperty("mongo.shell.command.table.limit")) + + script); + } + catch (IOException e) { + LOGGER.error("Can not write script in temp file", e); + return new InterpreterResult(Code.ERROR, e.getMessage()); + } + + InterpreterResult result = new InterpreterResult(InterpreterResult.Code.SUCCESS);; + + final DefaultExecutor executor = new DefaultExecutor(); + final ByteArrayOutputStream errorStream = new ByteArrayOutputStream(); + executor.setStreamHandler(new PumpStreamHandler(context.out, errorStream)); + executor.setWatchdog(new ExecuteWatchdog(commandTimeout)); //ExecuteWatchdog.INFINITE_TIMEOUT + + final CommandLine cmdLine = CommandLine.parse(getProperty("mongo.shell.path")); + cmdLine.addArgument("--quiet", false); + + if (!StringUtils.isEmpty(getProperty("mongo.server.username"))) { + cmdLine.addArgument("-u", false); + cmdLine.addArgument(getProperty("mongo.server.username"), false); + cmdLine.addArgument("-p", false); + cmdLine.addArgument(getProperty("mongo.server.password"), false); + + if (!StringUtils.isEmpty(getProperty("mongo.server.authentdatabase"))) { + cmdLine.addArgument("--authenticationDatabase", false); + cmdLine.addArgument(getProperty("mongo.server.authentdatabase"), false); + } + } + + cmdLine.addArgument(dbAddress, false); + cmdLine.addArgument(scriptFile.getAbsolutePath(), false); + + try { + executor.execute(cmdLine); + runningProcesses.put(context.getParagraphId(), executor); + } + catch (ExecuteException e) { + LOGGER.error("Can not run script in paragraph {}", context.getParagraphId(), e); + + final int exitValue = e.getExitValue(); + Code code = Code.ERROR; + String msg = errorStream.toString(); + if (exitValue == 143) { + code = Code.INCOMPLETE; + msg = msg + "Paragraph received a SIGTERM.\n"; + LOGGER.info("The paragraph {} stopped executing: {}", context.getParagraphId(), msg); + } + msg += "ExitValue: " + exitValue; + result = new InterpreterResult(code, msg); + } + catch (IOException e) { + LOGGER.error("Can not run script in paragraph {}", context.getParagraphId(), e); + result = new InterpreterResult(Code.ERROR, e.getMessage()); + } + finally { + FileUtils.deleteQuietly(scriptFile); + } + + return result; + } + + @Override + public int getProgress(InterpreterContext context) { + return 0; + } + + @Override + public void cancel(InterpreterContext context) { + stopProcess(context.getParagraphId()); + FileUtils.deleteQuietly(new File(getScriptFileName(context.getParagraphId()))); + } + + @Override + public Scheduler getScheduler() { + return SchedulerFactory.singleton().createOrGetParallelScheduler("mongo", 10); + } + + private String getScriptFileName(String paragraphId) { + return System.getProperty("java.io.tmpdir") + File.separator + + "zeppelin-mongo-" + paragraphId + ".js"; + } + + private void stopProcess(String paragraphId) { + if (runningProcesses.containsKey(paragraphId)) { + final Executor executor = runningProcesses.get(paragraphId); + final ExecuteWatchdog watchdog = executor.getWatchdog(); + watchdog.destroyProcess(); + } + } + +} diff --git a/zeppelin-mongodb-interpreter/src/main/resources/interpreter-setting.json b/zeppelin-mongodb-interpreter/src/main/resources/interpreter-setting.json new file mode 100644 index 00000000000..9ee4dd7e72a --- /dev/null +++ b/zeppelin-mongodb-interpreter/src/main/resources/interpreter-setting.json @@ -0,0 +1,72 @@ +[ + { + "group": "mongodb", + "name": "mongodb", + "className": "org.apache.zeppelin.mongodb.MongoDbInterpreter", + "properties": { + "mongo.shell.path": { + "envName": null, + "propertyName": "mongo.shell.path", + "defaultValue": "mongo", + "description": "MongoDB shell path", + "type": "string" + }, + "mongo.shell.command.table.limit": { + "envName": null, + "propertyName": "mongo.shell.command.table.limit", + "defaultValue": "1000", + "description": "Limit of documents displayed in a table", + "type": "number" + }, + "mongo.shell.command.timeout": { + "envName": null, + "propertyName": "mongo.shell.command.timeout", + "defaultValue": "60000", + "description": "MongoDB shell command timeout", + "type": "number" + }, + "mongo.server.host": { + "envName": null, + "propertyName": "mongo.server.host", + "defaultValue": "localhost", + "description": "MongoDB server host to connect to", + "type": "string" + }, + "mongo.server.port": { + "envName": null, + "propertyName": "mongo.server.port", + "defaultValue": "27017", + "description": "MongoDB server port to connect to", + "type": "number" + }, + "mongo.server.database": { + "envName": null, + "propertyName": "mongo.server.database", + "defaultValue": "test", + "description": "MongoDB database name", + "type": "string" + }, + "mongo.server.authentdatabase": { + "envName": null, + "propertyName": "mongo.server.authentdatabase", + "defaultValue": "", + "description": "MongoDB database name for authentication", + "type": "string" + }, + "mongo.server.username": { + "envName": null, + "propertyName": "mongo.server.username", + "defaultValue": "", + "description": "Username for authentication", + "type": "string" + }, + "mongo.server.password": { + "envName": null, + "propertyName": "mongo.server.password", + "defaultValue": "", + "description": "Password for authentication", + "type": "string" + } + } + } +] diff --git a/zeppelin-mongodb-interpreter/src/main/resources/shell_extension.js b/zeppelin-mongodb-interpreter/src/main/resources/shell_extension.js new file mode 100644 index 00000000000..876b3634459 --- /dev/null +++ b/zeppelin-mongodb-interpreter/src/main/resources/shell_extension.js @@ -0,0 +1,92 @@ +var _ZEPPELIN_TABLE_LIMIT_ = __ZEPPELIN_TABLE_LIMIT__; + +function flattenObject(obj, flattenArray) { + var toReturn = {}; + + for (var i in obj) { + if (!obj.hasOwnProperty(i)) continue; + + //if ((typeof obj[i]) == 'object') { + if (toString.call( obj[i] ) === '[object Object]' || + toString.call( obj[i] ) === '[object BSON]' || + (flattenArray && toString.call( obj[i] ) === '[object Array]')) { + var flatObject = flattenObject(obj[i]); + for (var x in flatObject) { + if (!flatObject.hasOwnProperty(x)) continue; + + toReturn[i + '.' + x] = flatObject[x]; + } + } else if (toString.call( obj[i] ) === '[object Array]') { + toReturn[i] = tojson(obj[i], null, true); + } else { + toReturn[i] = obj[i]; + } + } + return toReturn; +} + +function printTable(dbquery, fields, flattenArray) { + + var iterator = dbquery; + + if (toString.call( dbquery ) === '[object Array]') { + iterator = (function() { + var index = 0, + data = dbquery, + length = data.length; + + return { + next: function() { + if (!this.hasNext()) { + return null; + } + return data[index++]; + }, + hasNext: function() { + return index < length; + } + } + }()); + } + + // Flatten all the documents and get all the fields to build a table with all fields + var docs = []; + var createFieldSet = fields == null || fields.length == 0; + var fieldSet = fields ? [].concat(fields) : []; //new Set(fields); + + while (iterator.hasNext()) { + var doc = iterator.next(); + doc = flattenObject(doc, flattenArray); + docs.push(doc); + if (createFieldSet) { + for (var i in doc) { + if (doc.hasOwnProperty(i) && fieldSet.indexOf(i) === -1) { + fieldSet.push(i); + } + } + } + } + + fields = fieldSet; + + var header = "%table "; + fields.forEach(function (field) { header += field + "\t" }) + print(header.substring(0, header.length - 1)); + + docs.forEach(function (doc) { + var row = ""; + fields.forEach(function (field) { row += doc[field] + "\t" }) + print(row.substring(0, row.length - 1)); + }); +} + +DBQuery.prototype.table = function (fields, flattenArray) { + if (this._limit > _ZEPPELIN_TABLE_LIMIT_) { + this.limit(_ZEPPELIN_TABLE_LIMIT_); + } + printTable(this, fields, flattenArray); +} + +DBCommandCursor.prototype.table = DBQuery.prototype.table; + + diff --git a/zeppelin-mongodb-interpreter/src/test/java/org/apache/zeppelin/mongodb/MongoDbInterpreterTest.java b/zeppelin-mongodb-interpreter/src/test/java/org/apache/zeppelin/mongodb/MongoDbInterpreterTest.java new file mode 100644 index 00000000000..e64fd8e0032 --- /dev/null +++ b/zeppelin-mongodb-interpreter/src/test/java/org/apache/zeppelin/mongodb/MongoDbInterpreterTest.java @@ -0,0 +1,134 @@ +///* +// * Licensed to the Apache Software Foundation (ASF) under one or more +// * contributor license agreements. See the NOTICE file distributed with +// * this work for additional information regarding copyright ownership. +// * The ASF licenses this file to You under the Apache License, Version 2.0 +// * (the "License"); you may not use this file except in compliance with +// * the License. You may obtain a copy of the License at +// * +// * http://www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, +// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// * See the License for the specific language governing permissions and +// * limitations under the License. +// */ +//package org.apache.zeppelin.mongodb; +// +//import static org.junit.Assert.assertTrue; +// +//import java.io.File; +//import java.io.IOException; +//import java.nio.ByteBuffer; +//import java.util.Properties; +//import java.util.Scanner; +// +//import org.apache.commons.io.FileUtils; +//import org.apache.zeppelin.interpreter.InterpreterContext; +//import org.apache.zeppelin.interpreter.InterpreterOutput; +//import org.apache.zeppelin.interpreter.InterpreterOutputListener; +//import org.apache.zeppelin.interpreter.InterpreterResult; +//import org.apache.zeppelin.interpreter.InterpreterResult.Code; +//import org.apache.zeppelin.interpreter.InterpreterResultMessageOutput; +//import org.junit.Before; +//import org.junit.BeforeClass; +//import org.junit.Test; +// +///** +// * As there is no 'mongo' on the build platform, these tests simulates some basic behavior. +// * +// */ +//public class MongoDbInterpreterTest implements InterpreterOutputListener { +// +// private static final String SHELL_EXTENSION = +// new Scanner(MongoDbInterpreter.class.getResourceAsStream("/shell_extension.js"), "UTF-8") +// .useDelimiter("\\A").next(); +// +// private static final boolean IS_WINDOWS = System.getProperty("os.name") +// .startsWith("Windows"); +// +// private static final String MONGO_SHELL = System.getProperty("java.io.tmpdir") + +// File.separator + "mongo-test." + (IS_WINDOWS ? "bat" : "sh"); +// +// private final Properties props = new Properties(); +// private final MongoDbInterpreter interpreter = new MongoDbInterpreter(props); +// private final InterpreterOutput out = new InterpreterOutput(this); +// private final InterpreterContext context = new InterpreterContext("test", "test", +// null, null, null, null, null, null, null, null, null, out); +// +// private ByteBuffer buffer; +// +// @BeforeClass +// public static void setup() { +// // Create a fake 'mongo' +// final File mongoFile = new File(MONGO_SHELL); +// try { +// FileUtils.write(mongoFile, (IS_WINDOWS ? "@echo off\ntype \"%2%\"" : "cat \"$2\"")); +// FileUtils.forceDeleteOnExit(mongoFile); +// } +// catch (IOException e) { +// } +// } +// +// @Before +// public void init() { +// buffer = ByteBuffer.allocate(10000); +// props.put("mongo.shell.path", (IS_WINDOWS ? "" : "sh ") + MONGO_SHELL); +// props.put("mongo.shell.command.table.limit", "10000"); +// } +// +// @Test +// public void testSuccess() { +// final String userScript = "print('hello')"; +// +// final InterpreterResult res = interpreter.interpret(userScript, context); +// +// assertTrue("Check SUCCESS: " + res.message(), res.code() == Code.SUCCESS); +// +// try { +// out.flush(); +// } catch (IOException e) {} +// +// +// final String resultScript = new String(getBufferBytes()); +// +// final String expectedScript = SHELL_EXTENSION.replace( +// "__ZEPPELIN_TABLE_LIMIT__", interpreter.getProperty("mongo.shell.command.table.limit")) + +// userScript; +// +// // The script that is executed must contain the functions provided by this interpreter +// assertTrue("Check SCRIPT", resultScript.equals(expectedScript)); +// } +// +// @Test +// public void testBadConf() { +// props.setProperty("mongo.shell.path", "/bad/path/to/mongo"); +// final InterpreterResult res = interpreter.interpret("print('hello')", context); +// +// assertTrue(res.code() == Code.ERROR); +// } +// +// @Override +// public void onUpdateAll(InterpreterOutput interpreterOutput) { +// +// } +// +// @Override +// public void onAppend(int i, InterpreterResultMessageOutput interpreterResultMessageOutput, byte[] bytes) { +// buffer.put(bytes); +// } +// +// @Override +// public void onUpdate(int i, InterpreterResultMessageOutput interpreterResultMessageOutput) { +// +// } +// +// private byte[] getBufferBytes() { +// buffer.flip(); +// final byte[] bufferBytes = new byte[buffer.remaining()]; +// buffer.get(bufferBytes); +// return bufferBytes; +// } +// +//}