diff --git a/COPYING b/COPYING index d71ac74c..649d546d 100644 --- a/COPYING +++ b/COPYING @@ -1,4 +1,4 @@ - Copyright 2011 Vizrt + Copyright 2011-2013 Vizrt Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/README b/README index d5ca69e6..716059cf 100644 --- a/README +++ b/README @@ -1,72 +1,7 @@ -======== -Overview -======== -These are scripts to help template developers and administrators -manage various components of the Escenic Content Engine. The scripts -are experimental and not yet a part of the official distribution of -Escenic Content Engine from Vizrt Online. - -======================== -Project directory layout -======================== -The file structure ressembles where you'd typically put the various -scripts and configuration files. That's all it is, only for -documentational purposes :-) - -========================== -Features of the ece script -========================== -$ ece help -Usage: /usr/bin/ece [-t ] [-i ] - -DESCRIPTION - -t --type - The following types are available: - engine - The Escenic Content Engine, this is the default - and is the assumed type if none is specified. - search - A standalone search indexer and solr instance - rmi-hub - The RMI hub responsible for the internal - communication between the ECE instances. - - -i --instance - The type instance, such as editor1, web1 or search1 - - -p --publication - Needed only for updating publication resources - - -r --resource - Needed only for updating publication resources. - Must be one of: content-type, feature, layout, layout-group - image-version, menu - -v --verbose - Prints out debug statements, useful for debugging. - -The following commands are available: - applog the type's app server log - assemble runs the Assembly Tool *) - clean removes temporary files created by /home/torstein/bin/ece *) - deploy deploys the assembled EAR *) - help prints this help screen - kill uses force to stop the type - log the type's Log4J log - outlog the [ece#engine] script log (system out log) - restart restarts the type - start starts the type - status checks if the type is running - stop stops the type - threaddump write a thread dump to standard out (system out log) - update update publication resources - versions lists the ECE component versions - -*) only applicable if type is 'engine' - ========================== -Feedback +This repo/fork is obsolete ========================== -Hope you enjoy the ece /usr/bin and intit.d scripts. All feedback, -come hither! - --torstein@escenic.com - +Current development of ece-scripts is at: +https://github.com/escenic/ece-scripts diff --git a/etc/bash_completion.d/ece b/etc/bash_completion.d/ece index f22da95f..6e5d8bf8 100644 --- a/etc/bash_completion.d/ece +++ b/etc/bash_completion.d/ece @@ -1,33 +1,79 @@ -# auto completion for the /usr/bin/ece command. +# auto completion for the /usr/bin/ece command. Emacs: -*- sh -*- mode + _ece_commands() { local cur=${COMP_WORDS[COMP_CWORD]} local prev=${COMP_WORDS[COMP_CWORD-1]} - commands="applog assemble clean deploy help kill log outlog restart start status stop threaddump update versions" - resources="content-type feature layout layout-group image-version menu" + commands=" + applog + assemble + backup + clean + deploy + edit + flush + help + info + kill + list-deployments + list-instances + list-logs + list-publications + log + outlog + package + remove-old-log-files + restart start + status + stop + threaddump + top + update + versions + " + + options="-i --instance -p --publication -r --publication-resource + -t --type -u --user -w --password" + resources="content-type feature layout layout-group image-version menu + security root-section-parameters" types="engine search analysis rmi-hub" # default completions is the list of commands - completions=$commands - - long_options="--help" - - # TODO make educated guesses on these. - instances="" - publications="" + completions=$commands" "$options case "$prev" in - -t) + backup) + exclude_list=" + --exclude-binaries + --exclude-solr + --exclude-init + --exclude-conf + --exclude-db + --exclude-state + --exclude-multimedia + " + completions="$exclude_list $commands" + ;; + deploy) + completions="--file --uri $commands" + ;; + -t|--type) completions=$types ;; - -i) - completions=$instances + -i|--instance) + completions=$(ece -q list-instances) ;; - -p) - completions=$publications + -p|--publication) + local file=/usr/share/escenic/ece-scripts/common-ece.sh + if [ -r $file ]; then + source $file + completions=$(get_publication_list) + else + completions="" + fi ;; - -r) + -r|--publication-resource) completions=$resources ;; esac diff --git a/etc/bash_completion.d/ece-deploy b/etc/bash_completion.d/ece-deploy new file mode 100644 index 00000000..d1b7395f --- /dev/null +++ b/etc/bash_completion.d/ece-deploy @@ -0,0 +1,33 @@ +# auto completion for the ece-deploy command. -*- sh -*- + +_ece-deploy_commands() +{ + local cur=${COMP_WORDS[COMP_CWORD]} + local prev=${COMP_WORDS[COMP_CWORD-1]} + + options=" + --conf + --ear + --force + --list-deployments + --update-publication-resources + " + + # default completions is the list of options + case "$prev" in + --conf) + completions="http://" + ;; + --ear) + completions="http://" + ;; + *) + completions="$options" + ;; + esac + + COMPREPLY=( $(compgen -W "$completions" -- $cur) ) +} + +complete -o default -F _ece-deploy_commands ece-deploy + diff --git a/etc/bash_completion.d/ece-import b/etc/bash_completion.d/ece-import new file mode 100644 index 00000000..4ff3eafc --- /dev/null +++ b/etc/bash_completion.d/ece-import @@ -0,0 +1,32 @@ +# auto completion for the ece-import command. -*- sh -*- + +_ece-import_commands() +{ + local cur=${COMP_WORDS[COMP_CWORD]} + local prev=${COMP_WORDS[COMP_CWORD-1]} + + options=" + --directories-only + --escenic-group + --escenic-user + --import-archive + --name + --password + --publication + --uri + --user + --version + -V + -f + -n + -p + " + + # default completions is the list of options + completions="$options" + + COMPREPLY=( $(compgen -W "$completions" -- $cur) ) +} + +complete -o default -F _ece-import_commands ece-import + diff --git a/etc/bash_completion.d/ece-install b/etc/bash_completion.d/ece-install new file mode 100644 index 00000000..7ce24308 --- /dev/null +++ b/etc/bash_completion.d/ece-install @@ -0,0 +1,24 @@ +# auto completion for the ece-install command. -*- sh -*- + +_ece-install_commands() +{ + local cur=${COMP_WORDS[COMP_CWORD]} + local prev=${COMP_WORDS[COMP_CWORD-1]} + + options=" + --conf-file + --verbose + --version + -V + -f + -v + " + + # default completions is the list of options + completions="$options" + + COMPREPLY=( $(compgen -W "$completions" -- $cur) ) +} + +complete -o default -F _ece-install_commands ece-install + diff --git a/etc/bash_completion.d/vosa b/etc/bash_completion.d/vosa new file mode 100644 index 00000000..5e84442c --- /dev/null +++ b/etc/bash_completion.d/vosa @@ -0,0 +1,19 @@ +# auto completion for the vosa command. -*- sh -*- + +_vosa_commands() +{ + local cur=${COMP_WORDS[COMP_CWORD]} + local prev=${COMP_WORDS[COMP_CWORD-1]} + + commands=$( + vosa commands | cut -f3 -d' ' | cut -f1 -d':' + ) + + # default completions is the list of commands + completions="$commands" + + COMPREPLY=( $(compgen -W "$completions" -- $cur) ) +} + +complete -o default -F _vosa_commands vosa + diff --git a/etc/cron.d/periodic-check b/etc/cron.d/periodic-check new file mode 100644 index 00000000..9f3fae0b --- /dev/null +++ b/etc/cron.d/periodic-check @@ -0,0 +1,4 @@ +MAILTO=root + +* * * * * nagios /bin/sleep 20; if [ -x /usr/lib/periodic-check/main ]; then /usr/lib/periodic-check/main; fi + diff --git a/etc/cron.daily/remove-old-escenic-log-files b/etc/cron.daily/remove-old-escenic-log-files new file mode 100755 index 00000000..71d2a483 --- /dev/null +++ b/etc/cron.daily/remove-old-escenic-log-files @@ -0,0 +1,33 @@ +#! /usr/bin/env bash + +# cron script to remove old escenic related log files. It's safe to +# install this cron script on all servers, also the ones that don't +# have Escenic Content Engine or Escenic Analysis Engine installed. + +function remove_old_escenic_log_files() { + if [ ! -e /etc/init.d/ece ]; then + return + fi + + /etc/init.d/ece remove-old-log-files +} + +if [ -r /etc/default/ece ]; then + source /etc/default/ece + if [ ${remove_old_log_files-0} -eq 1 ]; then + remove_old_escenic_log_files + fi +fi + +# Added to take care of left over ear files in escenic cache by ece-install and ece-deploy +if [ -r /etc/escenic/ece.conf ]; then + source /etc/escenic/ece.conf + if [ ! -z "${cache_dir}" ] && [ -d "${cache_dir}" ] ; then + nice find "${cache_dir}" -maxdepth 1 -type f -mtime +5 -delete + fi + +fi + +# Now clean the /tmp directory as well +nice find /tmp -type f -mtime +3 -delete + diff --git a/etc/default/ece b/etc/default/ece index f7b2368b..ac916343 100644 --- a/etc/default/ece +++ b/etc/default/ece @@ -4,13 +4,13 @@ # instances. If you only have one # instance, use "default" as its # name. If you don't have any instances of that type on this host, # then just set the variable to an empty string. -engine_instance_list="app1 app2" -search_instance_list="search1" -analysis_instance_list="analysis1" +engine_instance_list="" +search_instance_list="" +analysis_instance_list="" # You only need one hub in an ECE cluster, so only set this one to -# true on one host. -enable_rmi_hub=true +# true on one host. 1 means on, 0 means off. +enable_rmi_hub=0 # The location of the ece script, default is /usr/bin/ece ece_script=/usr/bin/ece @@ -20,3 +20,7 @@ ece_script=/usr/bin/ece ece_unix_group=escenic ece_unix_user=escenic +# Do you want cron to remove (5 days) old log files every day? +# (default is yes, i.e. 1. Set 0 to turn this off) +remove_old_log_files=1 + diff --git a/etc/default/system-info b/etc/default/system-info new file mode 100644 index 00000000..ec078780 --- /dev/null +++ b/etc/default/system-info @@ -0,0 +1,3 @@ +# If you have blockdiag installed, but don't want system-info to +# generate diagrams, uncomment the following line: +# do_not_generate_diagram=1 diff --git a/etc/escenic/ece.conf b/etc/escenic/ece.conf index 2f3ad3bf..4f523aec 100644 --- a/etc/escenic/ece.conf +++ b/etc/escenic/ece.conf @@ -1,57 +1,88 @@ -# ece script configruation -*- sh -*- +# ece script configruation -*- conf -*- # version: $Revision: #1 $ $Date: 2011/02/18 $ # author: torstein@escenic.com # last update by: $Author: shud $ -######################################################################## +######################################################################### # Section I: Variables you probably want to change or at least verify # make sense. -######################################################################## +######################################################################### -######################################################################## +######################################################################### # Where the escenic content engine itself is installed -######################################################################## +######################################################################### ece_home=/opt/escenic/engine -######################################################################## +######################################################################### # Set this one if you're running more than one ECE instance on your # host. Usually, you'll set it to the instance name. If you're running # one ECE instance on your system, you may omit this one. -######################################################################## +######################################################################### # ece_server=rolling # ece_server_hostname=quanah.escenic.com -######################################################################## +######################################################################### +# Set this one if you wish to have a Nursery configuration layer +# special to the kind of enviornment you're running. An enviornment in +# this context is: development, test, staging and prodution. +######################################################################### +# ece_environment=production + +######################################################################### # Setting the java home, yes lowercase is correct ;-) -######################################################################## +######################################################################### java_home=/usr/lib/jvm/java-6-sun -######################################################################## +######################################################################### # Possible options are: tomcat, jboss, resin & oc4j -######################################################################## +######################################################################### appserver=tomcat -######################################################################## +######################################################################### +# App server port. If this variable is unset, ece will try to figure +# out the port number by itself. ece-.conf should override +# this when there's more than one ECE instance on the same host. +######################################################################### +appserver_port=8080 + +######################################################################### # Section II: Variables that are sensible defaults but may be changed # to accommodate the conventions or tastes of your OS or company. -######################################################################## +######################################################################### -######################################################################## +######################################################################### # Related to the ece script/ECE/hub/search -######################################################################## +######################################################################### assemblytool_home=/opt/escenic/assemblytool cache_dir=/var/cache/escenic -ece_security_configuration_dir=/etc/escenic/engine/common/security + +######################################################################### +# Configuration directories +######################################################################### +# The root of all Escenic configuration files (not only ECE) +escenic_conf_dir=/etc/escenic + +# The directory for the Escenic Analysis Engine/Stats' configuration +# files. Note that this is different from the EAE plugin for Content +# Studio, which has its configuration with the other ECE Nursery +# components. +analysis_conf_dir=$escenic_conf_dir/analysis +ece_security_configuration_dir=${escenic_conf_dir}/engine/common/security +rmi_hub_conf=${escenic_conf_dir}/rmi-hub + +######################################################################### +# Other important directories +######################################################################### +data_dir=/var/lib/escenic log_dir=/var/log/escenic -pid_dir=/var/run/escenic -rmi_hub_conf=/etc/escenic/rmi-hub +run_dir=/var/run/escenic rmi_hub_home=$ece_home/contrib/rmi-hub solr_home=/var/lib/escenic/solr tmp_dir=/tmp -######################################################################## +######################################################################### # When making a deployment, ece will default wise, deploy the EAR # generated with "ece assemble" on the application server. The # webapps included in the EAR can be filtered with the @@ -65,12 +96,19 @@ tmp_dir=/tmp # webapps listed here, will be deployed. # # Note: this is currently only supported when appserver=tomcat. -# +# # deploy_webapp_white_list="escenic-admin pbe" -######################################################################## +######################################################################### +# Memcached support. 'ece -i deploy' will per default add +# support for using memcached with your publication. If you wish to +# turn this off, you can set this value to 0. +######################################################################### +# enable_memcached_support=1 + +######################################################################### # Where to find the various application servers -######################################################################## +######################################################################### jboss_home=/opt/jboss oc4j_home=/opt/oc4j resin_home=/opt/resin @@ -81,48 +119,127 @@ tomcat_home=/usr/share/tomcat6 # binaries, just with different data. # tomcat_base=/opt/tomcat-editor1 -######################################################################## +###################################################################### +# If you've compiled or installed APR support (native Tomcat +# libraries, using the Apache APR library), you specify the install +# directory here, if not comment it out. +###################################################################### +apr_lib_dir=/usr/lib + +######################################################################### # JVM related (the Java Virtual Machine) -######################################################################## +######################################################################### -######################################################################## +######################################################################### # The minimum and maximum heap sizes for the Java process -######################################################################## -min_heap_size=2048m -max_heap_size=2048m +######################################################################### +min_heap_size=256m +max_heap_size=256m -######################################################################## +######################################################################### # The permgen size is the amount of memory the JVM sets aside to class # defintions. If you are experiencing constant OutOfMemory # exceptions/errors and are using EAR deployment, you may want to # increase these. -######################################################################## +######################################################################### min_permgen_size=64m max_permgen_size=256m -enable_heap_dump=0 enable_remote_debugging=0 remote_debugging_port=5005 enable_remote_monitoring=0 remote_monitoring_port=5792 -##################################################################### +######################################################################### +# Settings related to the JVM garbage collection, don't touch these if +# you're not sure of what you're doing and have tested your changes +# properly. Note that ece takes care of the GC logging, so you don't +# have to cater for that here. +######################################################################### +jvm_gc_settings=" +-XX:+CMSClassUnloadingEnabled +-XX:+CMSIncrementalPacing +-XX:+CMSPermGenSweepingEnabled +-XX:+ExplicitGCInvokesConcurrent +-XX:+UseCMSInitiatingOccupancyOnly +-XX:+UseConcMarkSweepGC +-XX:CMSInitiatingOccupancyFraction=65 +-XX:InitialCodeCacheSize=50m +-XX:MaxNewSize=250m +-XX:NewSize=250m +-XX:ParallelGCThreads=1 +-XX:ReservedCodeCacheSize=50m +-XX:SurvivorRatio=1 +" + +###################################################################### +# If you want ECE/EAE to use an HTTP proxy to access the internet, set +# the parameters you need here and remove the rest. Default is no HTTP +# proxy. See +# http://docs.oracle.com/javase/6/docs/technotes/guides/net/proxies.html +# for more details on the available options here. +###################################################################### +# jvm_proxy_settings=" +# -Dhttp.proxyHost=proxyserver +# -Dhttp.proxyPort=3128 +# -Dhttp.nonProxyHosts=localhost +# -Dhttps.proxyHost=proxyserver +# -Dhttps.proxyPort=3128 +# " + +###################################################################### +# This overrides the default connection settings of all classes that +# directly or indirectly open TCP connections inside the JVM. +# +# To ensue more stable connections to EAE, we set max timeouts +# here. If you're not using EAE, you probably don't need these and can +# comment out the variable. +# +# The values are in milliseconds and are set to 5 seconds (5000 +# milliseconds). +jvm_connection_settings=" + -Dsun.net.client.defaultConnectTimeout=1000 + -Dsun.net.client.defaultReadTimeout=5000 +" + +###################################################################### # Java will use IPv6 if it's available on the OS. This however, can # cause problems with applications that insists on using IPv4 # sockets. # # If e.g. Tomcat cannot create a socket on a given port (which is not # taken), you may need to set this one to '1'. Default is '0'. -##################################################################### +###################################################################### force_ipv4=0 -##################################################################### -# Is this a production system? -##################################################################### -is_production=1 - -##################################################################### -# Ask the JVM to create a heap dump when/if it crashes. -##################################################################### +###################################################################### +# Ask the JVM to create a heap dump when/if it crashes. +###################################################################### enable_heap_dump=1 heap_dump_dir=/var/crash/escenic + +###################################################################### +# Where 'ece -i backup' will put its snapshot +# backups. Default is /var/backups/escenic. +###################################################################### +backup_dir=/var/backups/escenic + +###################################################################### +# When removing old log files, how old should the oldest file be? +###################################################################### +old_log_file_max_age_in_days=5 + +###################################################################### +# HTTP authentication for the escenic-admin webapp (default is no HTTP +# auth) +###################################################################### +# escenic_admin_http_user= +# escenic_admin_http_password= + +###################################################################### +# HTTP authentication for the build server (default is no HTTP auth), +# this only used for when you use 'ece deploy' with a URI for an EAR +# a the build server. +###################################################################### +# builder_http_user= +# builder_http_password= diff --git a/etc/init.d/ece b/etc/init.d/ece index dc2a515c..0c358b76 100755 --- a/etc/init.d/ece +++ b/etc/init.d/ece @@ -5,8 +5,8 @@ ### BEGIN INIT INFO # Provides: ece -# Required-Start: $remote_fs $network $syslog -# Required-Stop: $remote_fs $network $syslog +# Required-Start: $local_fs $network $syslog +# Required-Stop: $local_fs $network $syslog # Default-Start: 2 3 4 5 # Default-Stop: 0 1 6 # Short-Description: Escenic Content Engine @@ -25,31 +25,41 @@ # would be: update-rc.d ece defaults ####################################################################### +if [ $(whoami) != "root" ]; then + echo "Sorry, you must be root for running $0" + exit 1 +fi + ####################################################################### # Default values ####################################################################### dir_list=" /var/cache/escenic +/var/backups/escenic /var/crash/escenic /var/log/escenic +/var/lib/escenic /var/run/escenic " ece_script=/usr/bin/ece -# the values above may be overidden a file named the same as this -# init.d script, located in: +# The values above may be overidden a file named the same as this +# init.d script. This init.d configuration must also hold the +# variables controlling which ECE instances to start. The list of +# locations per default caters (at least) for Debian, RedHat & Gentoo +# based systems: conf_file_location_list=" /etc/default /etc/conf.d -/tmp +/etc/sysconfig " function read_conf_file() { for el in $conf_file_location_list; do - if [ -r $el/`basename $0` ]; then - source $el/`basename $0` + if [ -r $el/ece ]; then + source $el/ece found_conf=1 break fi @@ -67,8 +77,10 @@ function ensure_dirs_are_ok() if [ ! -d $el ]; then mkdir -p $el fi - chown -R $ece_unix_user:$ece_unix_group $el + + chown --changes $ece_unix_user:$ece_unix_group $el done + } function ensure_ece_script_is_ok() @@ -85,7 +97,7 @@ function ensure_ece_script_is_ok() function execute_command() { - if [ $enable_rmi_hub = "true" ]; then + if [ $enable_rmi_hub -eq 1 ]; then su - $ece_unix_user -c "$ece_script -t rmi-hub $1" fi diff --git a/etc/init.d/vosa b/etc/init.d/vosa new file mode 100644 index 00000000..575c5009 --- /dev/null +++ b/etc/init.d/vosa @@ -0,0 +1,114 @@ +#! /usr/bin/env bash + +# version: $Revision: #1 $ $Date: 2011/02/18 $ +# author: mogsie@vizrt.com + +### BEGIN INIT INFO +# Provides: vosa +# Required-Start: $remote_fs $network $syslog +# Required-Stop: $remote_fs $network $syslog +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Vizrt Online Components +### END INIT INFO + +################################################################# +# Usage: +# * copy this script to /etc/init.d/vosa +# +# * copy its configuration file to /etc/default/vosa (or +# * /etc/conf.d/vosa on Gentoo based disitributions) +# +# * make sure it's executable, chmod +x /etc/init.d/vosa +# +# * add it to the desired runlevels. On Debian based systems this +# would be: update-rc.d vosa defaults +####################################################################### + +if [ $(whoami) != "root" ] ; then + echo "Sorry, you must be root to run $0" + exit 1 +fi + +####################################################################### +# Default values +####################################################################### +dir_list=" +/var/run/vizrt/vosa +" + +vosa_instance_list= +for a in /etc/vizrt/vosa/enabled.d/* ; do + if [ -L "$a" -a -r "$a" ] ; then + vosa_instance_list="$vosa_instance_list $(basename "$a")" + fi +done + +vosa_script=/usr/bin/vosa + +# The values above may be overidden a file named the same as this +# init.d script. This init.d configuration must also hold the +# variables controlling which vosa instances to start. The list of +# locations per default caters (at least) for Debian, RedHat & Gentoo +# based systems: +conf_file_location_list=" +/etc/default +/etc/conf.d +/etc/sysconfig +" + +function read_conf_file() +{ + for el in $conf_file_location_list; do + if [ -r $el/`basename $0` ]; then + source $el/`basename $0` + found_conf=1 + break + fi + done + +# if [ -z $found_conf ]; then +# echo "Couldn't find configuration for $0, exiting :-(" +# exit 1 +# fi +} + +function ensure_dirs_are_ok() +{ + for el in $dir_list; do + if [ ! -d $el ]; then + mkdir -p $el + fi +# don't change owner; root is fine... +# chown --changes -R $vosa_unix_user:$vosa_unix_group $el + done +} + +function ensure_vosa_script_is_ok() +{ + if [ ! -r $vosa_script ]; then + echo "Couldn't read $vosa_script" + exit 1 + elif [ ! -x $vosa_script ]; then + echo "$vosa_script needs to be executable (do: chmod +x $vosa_script)" + exit 1 + fi + +} + +function execute_command() +{ + for el in $vosa_instance_list; do + $vosa_script -i $el $1 + done +} + +if [ $1 ]; then + # todo: whitelist to start, stop, status. + read_conf_file + ensure_vosa_script_is_ok + ensure_dirs_are_ok + execute_command $1 +else + echo "Usage: `basename $0` " +fi diff --git a/etc/logrotate.d/tomcat b/etc/logrotate.d/tomcat new file mode 100644 index 00000000..95e40834 --- /dev/null +++ b/etc/logrotate.d/tomcat @@ -0,0 +1,9 @@ +/var/log/escenic/*.out{ + copytruncate + daily + rotate 7 + compress + missingok + size 5M +} + diff --git a/etc/munin/plugins/indexer_running b/etc/munin/plugins/indexer_running new file mode 100755 index 00000000..405ce9f5 --- /dev/null +++ b/etc/munin/plugins/indexer_running @@ -0,0 +1,37 @@ +#!/bin/bash +# + +#. $MUNIN_LIBDIR/plugins/plugin.sh +info_file="/var/cache/periodic-check/indexer_running" +state=$(cat $info_file | cut -f 3 -d ' ') +age=$(cat $info_file | cut -f 4 -d ' ') + +if [ "$1" = "config" ]; then + echo 'graph_title indexing delay' + echo "graph_args --base 1000 -r --lower-limit 0 " + echo 'graph_vlabel Minutes' + echo 'graph_order indexing_delay' +# echo 'graph_scale no' + echo 'graph_info Shows the time to get a document indexed by Escenic search. This is basically time difference between update time of head-tail file and entryUpdated field in SearchIndex table.' + echo 'graph_category system' + echo 'graph_period second' + echo "indexing_delay.label time to index current document" + echo "indexing_delay.draw LINE2" + echo "indexing_delay.min 0" + echo "indexing_delay.type GAUGE" + echo "indexing_delay.info time in $site in Bytes" + exit 0; +fi + +# Note: Counters/derive need to report integer values. Also we need +# to avoid 10e+09 and the like %.0f should do this. + +cur_epoch=$(date +%s) +sample_epoch=$(cat "$info_file" | head -n 1| cut -f 1 -d ' ') +#add 90 seconds to compensate freshness. +sample_epoch=$(echo "$sample_epoch + 90"| bc) + +if [ "$sample_epoch" -gt "$cur_epoch" ] && [ "$state" == "yes" ] && [ "$age" -ge 0 ]; then + printf "indexing_delay.value $age\n" +fi + diff --git a/etc/munin/plugins/site_length b/etc/munin/plugins/site_length new file mode 100755 index 00000000..2c4ce04b --- /dev/null +++ b/etc/munin/plugins/site_length @@ -0,0 +1,52 @@ +#!/bin/bash +# + +#. $MUNIN_LIBDIR/plugins/plugin.sh +info_file="/var/cache/periodic-check/site_length" +sites=($(cat $info_file | cut -f 2 -d ' ')) +lengths=($(cat $info_file | cut -f 3 -d ' ')) + +if [ "$1" = "autoconf" ]; then + if [ -r /proc/stat ]; then + echo yes + exit 0 + else + echo no + exit 0 + fi +fi + +if [ "$1" = "config" ]; then + + echo 'graph_title sites length' + echo "graph_order " ${sites[@]//./_} + echo "graph_args --base 1000 -r --lower-limit 0 " + echo 'graph_vlabel Bytes' +# echo 'graph_scale no' + echo 'graph_info This graph shows length of different pages.' + echo 'graph_category system' + echo 'graph_period second' + for site in "${sites[@]}"; do + echo "${site//./_}.label $site" + echo "${site//./_}.draw LINE2" + echo "${site//./_}.min 0" + echo "${site//./_}.type GAUGE" + echo "${site//./_}.info length/size of $site in Bytes" + done + exit 0 +fi + +# Note: Counters/derive need to report integer values. Also we need +# to avoid 10e+09 and the like %.0f should do this. + +cur_epoch=$(date +%s) +sample_epoch=$(cat "$info_file" | head -n 1| cut -f 1 -d ' ') +#add 90 seconds to compensate freshness. +sample_epoch=$(echo "$sample_epoch + 90"| bc) + +if [ "$sample_epoch" -gt "$cur_epoch" ]; then + for i in $(seq 1 ${#sites[@]}); do + printf "%s.value %.0f\n" ${sites[$i-1]//./_} ${lengths[$i-1]} + done +fi + diff --git a/etc/puppet/manifests/global-variables.pp b/etc/puppet/manifests/global-variables.pp new file mode 100644 index 00000000..badaf59e --- /dev/null +++ b/etc/puppet/manifests/global-variables.pp @@ -0,0 +1,13 @@ +# global variables +$control_host = "control" + +$apt_vizrt_user = "" +$apt_vizrt_password = "" +$apt_proxy_host = "$control_host" + +$db_user = "ece5user" +$db_vip = "" + +$privileged_ip_list = "" + +$mobilize_vip = "" diff --git a/known-issues.org b/known-issues.org new file mode 100644 index 00000000..3e6ce79a --- /dev/null +++ b/known-issues.org @@ -0,0 +1,26 @@ +#+TITLE: Known Issues +#+AUTHOR: Vizrt Online / The SaaS Team + +* ece-install +** 2013-01-17 cron script removes all 15 day old files on the system +- *Fixed in Version* :: 1.0-2013-01-17-468 + +- *Versions affected* :: 1.0-2013-01-12-x => 1.0-2013-01-17-x + +If you have installed and run either the =ece-install= all-in-one, +presentation or editorial profile between *2013-01-12* +and *2013-01-17*, you have encountered this issue. + +If you have =escenic-content-engine-installer= of any of the versions +affected, first remove the cron job if it's present: +#+BEGIN_SRC sh +# rm /etc/cron.daily/remove-old-escenic-cache-files +#+END_SRC + +Then upgrade to the latest version of =ece-install= so that you're +sure new runs of it will set up a proper cron job. If you're using +our APT repository at apt.vizrt.com, this as easy as: +#+BEGIN_SRC sh +# apt-get update +# apt-get install escenic-content-engine-scripts +#+END_SRC diff --git a/usr/bin/ece b/usr/bin/ece index 24575a55..cab6ee07 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -6,20 +6,41 @@ # # echo comments and suggestions > tkj@vizrt.com +###################################################################### +# Needs to be here to bootstrap the ece script itself when it looks +# for the ece[-[a-z0-9]*].conf files. +# +# There is one to avoid this bootstrap dependency, and that is to set +# the ECE_CONF_LOCATIONS environment variable, however this is quite +# uncommon and generally speaking not recommended. +###################################################################### +escenic_conf_dir=/etc/escenic ###################################################################### # Script defaults, my be overriden in any of the configration files ###################################################################### -verbose=0 force_ipv4=0 +quiet=0 +everything_but_the_kitchen_sink=0 ###################################################################### # Dear user, don't touch anyting in this script, especially below this # line :-) ###################################################################### +ece_scripts_version="straight-from-github" type=engine instance=default command="" +backup_exclude_binaries=0 +backup_exclude_conf=0 +backup_exclude_db=0 +backup_exclude_init=0 +backup_exclude_multimedia=0 +backup_exclude_solr=0 +backup_exclude_state=0 +wget_appserver_auth="" +curl_appserver_auth="" +log="" type_list=" engine @@ -27,13 +48,6 @@ search rmi-hub " -common_settings_list=" -ece_home -java_home -log_dir -pid_dir -" - hub_required_fields=" ece_server_hostname rmi_hub_conf @@ -44,116 +58,124 @@ engine_required_fields=" appserver assemblytool_home cache_dir +data_dir +escenic_conf_dir ece_home ece_security_configuration_dir enable_heap_dump enable_remote_debugging enable_remote_monitoring -is_production java_home solr_home +log_dir +run_dir +tmp_dir " search_required_fields=" appserver java_home solr_home +tmp_dir " analysis_required_fields=" appserver java_home +tmp_dir " -#################################################################### -# Will exit the ece execution if the last operation failed. While -# failing, it will print the message passed to the function. -#################################################################### -function exit_on_error() -{ - if [ $? -eq 1 ]; then - print $1 "FAILED, exiting :-(" - exit 1 - fi -} -#################################################################### -# debug/verbose method -#################################################################### -function debug() -{ - if [ $verbose -eq 1 ]; then - print $@ - fi -} - -function print() -{ - echo "$id" $@ -} - -function log() -{ - echo "$id" `date` $@ 2>/dev/null >> $log_file -} - -function verify_that_directory_and_file_are_writeable() -{ - dir=`dirname $1` - if [ ! -e $dir ]; then - print $dir " doesn't exist" - exit 1 - fi - if [ ! -w $dir ]; then - print $dir " isn't writable for user $USER" - exit 1 - fi +## Bootstrapping, load files from /usr/share/escenic/ece-scripts The +## method will first try to be smart, in case the user has copied the +## ece-scripts somewhere else., e.g.: moved everything to ~/ece-scrpts +## or /opt/escenic/ece-scripts, this should also work. +function init() { + # first, try to be nice + local dir=$(dirname $0)/../share/escenic/ece-scripts + + # then check the standard location + if [ ! -d $dir ]; then + dir=/usr/share/escenic/ece-scripts + fi + + if [ -d $dir ]; then + # load common libraries + common_libraries="common-bashing.sh common-io.sh common-os.sh common-ece.sh" + for el in $common_libraries; do + source $dir/${el} + done - if [ -e $1 ]; then - if [ ! -w $1 ]; then - print $1 "exists, " - print "but isn't write-able for user $USER" - print "check the permissions and that $USER is the correct one" - print "to run "`basename $0` - exit 1 - fi - fi + # load ece-install modules + for el in $dir/ece.d/*.sh; do + source $el + done + else + echo "I cannot find $(basename $0)'s dependencies, exiting :-(" + exit 1 + fi } #################################################################### # Ensures that all required fields are set. Will report all missing # required fields before failing. #################################################################### -function ensure_that_required_fields_are_set() -{ +function ensure_that_required_fields_are_set() { requirements_failed=0 - + for el in $@; do if [ -n "$(eval echo $`echo $el`)" ]; then continue fi - - print "You need to specifiy '$el' in your `basename $0`.conf" + + print "You need to specifiy '$el' in one of ${ece_conf_files_read[@]}" requirements_failed=1 done - if [ $requirements_failed -eq 1 ]; then + if [ "$requirements_failed" -eq 1 ]; then exit 1 fi } -function read_conf_file() -{ +ece_conf_files_read=() + +function read_conf_file() { for el in $conf_file_location_list; do if [ -r $el/$1 ]; then debug "found $1 in $el, reading it" source $el/$1 + ece_conf_files_read=($el/$1 ${ece_conf_files_read}) break fi done } -function set_common_settings() -{ +## If the system is installed using the recommended paths, the method +## will return a list of the instances configured in +## ${escenic_conf_dir}/ece-*.conf +function get_instance_list() { + local allowed_types="engine analysis changelogd search rmi-hub" + instance_list="" + for el in $(\ls /etc/escenic/ece-*.conf 2>/dev/null); do + + local instance=$(basename $el .conf) + instance=${instance##ece-} + + for ele in $allowed_types; do + if [[ $instance == $ele ]]; then + instance="" + continue + fi + + local away="${ele}-" + instance=${instance##${away}} + done + instance_list="$instance_list $instance" + done + + echo $instance_list +} + +function set_common_settings() { # This behaviour can be overridden by specifying a list of # locations in the environment variable ECE_CONF_LOCATIONS if [ -n "$ECE_CONF_LOCATIONS" ] ; then @@ -161,43 +183,79 @@ function set_common_settings() else conf_file_location_list=" `dirname $0` - /etc/escenic/$type/instance/$instance - /etc/escenic/$type/host/`echo $HOSTNAME | tr '[A-Z]' '[a-z]'` - /etc/escenic/$type/common - /etc/escenic/$type - /etc/escenic + ${escenic_conf_dir}/$type/instance/$instance + ${escenic_conf_dir}/$type/host/`echo $HOSTNAME | tr '[A-Z]' '[a-z]'` + ${escenic_conf_dir}/$type/common + ${escenic_conf_dir}/$type + ${escenic_conf_dir} `dirname $0`/../etc " fi # main configuration file, may be overridden in the type and # instance specific ones. - read_conf_file `basename $0`.conf + read_conf_file $(basename $0).conf - if [ $instance = "default" ]; then - log_file=$log_dir/$type.out - pid_file=$pid_dir/$type.pid - else - log_file=$log_dir/$type-$instance.out - pid_file=$pid_dir/$type-$instance.pid - fi + log=$log_dir/$instance.out + pid_file=$run_dir/$instance.pid + + gc_log=${log_dir}/${instance}-gc.log - log_file_list=" + log4j_file_list=" + $log_dir/${HOSTNAME}-${instance}-messages + $log_dir/${instance}-messages $log_dir/messages $log_dir/Escenic-error.log " +} + +function set_type_settings() { + # optional: possible to have type specific conf file, + # will take precedences over the common one. + read_conf_file $(basename $0)-$type.conf +} + +function set_type_pid() { + if [ "$(uname)" == "Linux" ]; then + ps_options="auxww" + else + ps_options="-ef" + fi + + # Get a hold of the PID of the process. Although we've got the PID + # file, we stil get the PID from the OS here as we used it for + # sanity checking later on, see stop_type(). + if [ "$type" == "rmi-hub" ]; then + # we need to be able to differentiate between an ECE instance + # and an rmi hub, for this we use java.security.policy which + # the hub doesn't have. + type_pid=`ps $ps_options | grep -v java.security.policy | \ + awk "/Djava.rmi.server.hostname=$ece_server_hostname / && \ + !/awk/"' {print $2}'` + else + type_pid=`ps $ps_options | awk "/Descenic.server=$ece_server / && !/awk/"' {print $2}'` + fi + + debug "type_pid is now=$type_pid" +} + +function set_type_port() { host=localhost - if [ $appserver != "tomcat" ]; then + # port set in ece[-instance].conf takes precedence + if [ -n "${appserver_port}" ]; then + port=${appserver_port} + debug "appserver_port set in .conf files=${port}" + elif [ "$appserver" != "tomcat" ]; then debug "Only tomcat is supported for reading host/port right now, " debug "trying to make an educated guess" port=8080 else if [ -r $tomcat_base/logs/catalina.out ]; then # for tomcat6-user packaged tomcats (and perhaps others) - out_log=$tomcat_base/logs/catalina.out + out_log=$log_dir/${instance_name}-tomcat-catalina.out else - out_log=$log_file + out_log=$log fi if [ -r $out_log ]; then @@ -207,59 +265,34 @@ function set_common_settings() cut -d'-' -f2 ) fi - fi - -} -function set_type_settings() -{ - # optional: possible to have type specific conf file, - # will take precedences over the common one. - read_conf_file `basename $0`-$type.conf + debug "set_type_port=$port" } -function set_type_pid() -{ - # Get a hold of the PID of the process. Although we've got the PID - # file, we stil get the PID from the OS here as we used it for - # sanity checking later on, see stop_type(). - if [ $type = "rmi-hub" ]; then - # we need to be able to differentiate between an ECE instance - # and an rmi hub, for this we use java.security.policy which - # the hub doesn't have. - type_pid=`ps -ef | grep -v java.security.policy | \ - awk "/Djava.rmi.server.hostname=$ece_server_hostname / && \ - !/awk/"' {print $2}'` - else - type_pid=`ps -ef | awk "/Descenic.server=$ece_server / && !/awk/"' {print $2}'` - fi - - debug "type_pid is now=$type_pid" -} - -function set_instance_settings() -{ +function set_instance_settings() { # optional: possible to have instance specific conf files, # these will take precedence over the other two - read_conf_file `basename $0`-$instance.conf - read_conf_file `basename $0`-$type-$instance.conf + read_conf_file $(basename $0)-$instance.conf + read_conf_file $(basename $0)-$type-$instance.conf # at this point in the script flow, we have now read all the # possible combinations of conf files (common, type and instance) # and can now ensure that required fields are set and apply them. - if [ $type = "rmi-hub" ]; then + if [ "$type" == "rmi-hub" ]; then ensure_that_required_fields_are_set $hub_required_fields - elif [ $type = "search" ]; then + elif [ "$type" == "search" ]; then ensure_that_required_fields_are_set $search_required_fields - elif [ $type = "engine" ]; then + elif [ "$type" == "engine" ]; then ensure_that_required_fields_are_set $engine_required_fields + elif [ "$type" == "analysis" ]; then + ensure_that_required_fields_are_set $analysis_required_fields fi # We respect ece_server if it's set in any of the configuration # files. If it's not set there, it sets some sensible defaults. - if [ -z $ece_server ]; then - if [ $instance = "default" ]; then + if [ -z "$ece_server" ]; then + if [ "$instance" == "default" ]; then ece_server=${HOSTNAME} else ece_server=${HOSTNAME}-${instance} @@ -267,9 +300,9 @@ function set_instance_settings() fi set_type_pid - + # if type is rmi-hub, we don't' need more configuration. - if [ $type = "rmi-hub" ]; then + if [ "$type" == "rmi-hub" ]; then return fi @@ -282,74 +315,121 @@ function set_instance_settings() # * java.awt.headless is to avoid potential problems with graphics # handling/generation, causing 0x0 bitmaps etc. # * java.security for configuring the Java security framework with ECE. - ece_args="-Descenic.server=$ece_server\ - -Dsolr.solr.home=$solr_home\ - -Djava.awt.headless=true\ - -Djava.security.auth.login.config=$ece_security_configuration_dir/jaas.config\ - -Djava.security.policy=$ece_security_configuration_dir/java.policy" + # * garbage collection log: this is paramount to keep an eye on + # when running in production. + ece_args=" + -Descenic.server=$ece_server + -Djava.awt.headless=true + -Djava.security.auth.login.config=$ece_security_configuration_dir/jaas.config + -Djava.security.policy=$ece_security_configuration_dir/java.policy + -Dsolr.solr.home=$solr_home + -XX:+PrintGCDetails + -XX:+PrintGCTimeStamps + -Xloggc:${gc_log} + " + + if [ "$jvm_gc_settings" ]; then + ece_args=$ece_args" "$jvm_gc_settings + fi + + jvm_rolling_gc=${jvm_rolling_gc-1} + if [ "$jvm_rolling_gc" == 1 ]; then + gc_rolling_args=" + -XX:+UseGCLogFileRotation + -XX:NumberOfGCLogFiles=${number_of_gc_log_files-5} + -XX:GCLogFileSize=${gc_log_file_size-5m} + " + ece_args=$ece_args" "$gc_rolling_args + fi - if [ $instance != "default" ]; then + if [ "$jvm_proxy_settings" ]; then + ece_args=$ece_args" "$jvm_proxy_settings + fi + + if [ "$jvm_connection_settings" ]; then + ece_args=$ece_args" "$jvm_connection_settings + fi + + if [ "$instance" != "default" ]; then ece_args=$ece_args" -Dcom.escenic.instance=$instance" fi - if [ $ece_server_hostname ]; then + if [ -n "$ece_environment" ]; then + ece_args=$ece_args" -Dcom.escenic.environment=${ece_environment}" + fi + + if [ "$ece_server_hostname" ]; then ece_args=$ece_args" -Djava.rmi.server.hostname=$ece_server_hostname" fi - if [ $apr_lib_dir ]; then + if [ "$apr_lib_dir" ]; then ece_args=$ece_args" -Djava.library.path=$apr_lib_dir" fi - if [ $min_heap_size ]; then + if [ "$min_heap_size" ]; then ece_args=$ece_args" -Xms$min_heap_size" fi - if [ $max_heap_size ]; then + if [ "$max_heap_size" ]; then ece_args=$ece_args" -Xmx$max_heap_size" fi - if [ $min_permgen_size ]; then + if [ "$min_permgen_size" ]; then ece_args=$ece_args" -XX:PermSize=$min_permgen_size" fi - if [ $max_permgen_size ]; then + if [ "$max_permgen_size" ]; then ece_args=$ece_args" -XX:MaxPermSize=$max_permgen_size" fi - - # settings specific to a production environment - if [ $is_production -eq 1 ]; then - ece_args=$ece_args" -server" - fi - - if [ $jvm_encoding ]; then + + if [ "$jvm_encoding" ]; then ece_args=$ece_args" -Dsun.jnu.encoding=$jvm_encoding" ece_args=$ece_args" -Dfile.encoding=$jvm_encoding" fi - if [ $enable_remote_debugging -eq 1 ]; then + if [ "$analysis_conf_dir" ]; then + ece_args=$ece_args" -Dcom.escenic.eae.config=${analysis_conf_dir}" + fi + + if [ "$enable_remote_debugging" -eq 1 ]; then ece_args=$ece_args" -Xdebug -Xnoagent -Djava.compiler=NONE \ -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=$remote_debugging_port" - fi + fi - if [ $enable_remote_monitoring -eq 1 ]; then + if [ "$enable_remote_monitoring" -eq 1 ]; then ece_args=$ece_args" -Dcom.sun.management.jmxremote" ece_args=$ece_args" -Dcom.sun.management.jmxremote.authenticate=false" ece_args=$ece_args" -Dcom.sun.management.jmxremote.ssl=false" - ece_args=$ece_args" -Dcom.sun.management.jmxremote.port=$remote_monitoring_port" + ece_args=$ece_args" -Dcom.sun.management.jmxremote.port=$remote_monitoring_port" fi - if [ $enable_heap_dump -eq 1 ]; then + if [ "$enable_heap_dump" -eq 1 ]; then ece_args=$ece_args" -XX:+HeapDumpOnOutOfMemoryError" ece_args=$ece_args" -XX:HeapDumpPath=$heap_dump_dir" fi - if [ $force_ipv4 -eq 1 ]; then + if [ "$force_ipv4" -eq 1 ]; then ece_args=$ece_args" -Djava.net.preferIPv4Stack=true" fi - + + if [[ -n "$escenic_admin_http_user" && -n "$escenic_admin_http_password" ]]; then + wget_appserver_auth=" + --http-user $escenic_admin_http_user + --http-password $escenic_admin_http_password + " + curl_appserver_auth="-u ${escenic_admin_http_user}:${escenic_admin_http_password}" + fi + + if [[ -n "$builder_http_user" && -n "$builder_http_password" ]]; then + wget_builder_auth=" + --http-user $builder_http_user + --http-password $builder_http_password + " + fi + # Resin needs some more arguments as its XML parser is not # compatible with ECE. - if [ $appserver = "resin" ]; then + if [[ "$appserver" == "resin" ]]; then ece_args="$ece_args\ -Dorg.xml.sax.driver=org.apache.xerces.parsers.SAXParser -Djavax.xml.stream.XMLInputFactory=com.sun.xml.stream.ZephyrParserFactory @@ -358,8 +438,8 @@ function set_instance_settings() -Djavax.xml.transform.TransformerFactory=org.apache.xalan.processor.TransformerFactoryImpl" debug "resin_home=$resin_home" - elif [ $appserver = "tomcat" ]; then - if [ -z $tomcat_base ]; then + elif [ "$appserver" == "tomcat" ]; then + if [ -z "$tomcat_base" ]; then tomcat_base=$tomcat_home else export CATALINA_BASE=$tomcat_base @@ -371,644 +451,341 @@ function set_instance_settings() export ECE_HOME=$ece_home export JAVA_OPTS="$JAVA_OPTS $ece_args" export JAVA_HOME=$java_home - + debug ECE_HOME=$ECE_HOME debug JAVA_HOME=$JAVA_HOME debug JAVA_OPTS=$JAVA_OPTS - } -function sanity_check() -{ - verify_that_directory_and_file_are_writeable $log_file - verify_that_directory_and_file_are_writeable $pid_file - - if [ -z "$command" ]; then - print "You must specificy a command, see 'ece help'." +function sanity_check() { + if [ "$(whoami)" == "root" ]; then + print "Sorry, you cannot be root when running $(basename $0)" + print "The root user can only use /etc/init.d/ece" exit 1 fi -} -function deploy() -{ - ear=$cache_dir/engine.ear - if [ ! -e $ear ]; then - print "$ear does not exist. Did you run '"`basename $0`" assemble'?" - exit 1 + if [ "$(echo $command)" == "list-instances" ]; then + return fi - - # extract EAR to a temporary area - dir=$tmp_dir/`basename $0`-`date +%s` - (mkdir -p $dir && cd $dir && jar xf $ear) - exit_on_error "extracting $ear to $dir" - - print "Deploying $ear on $appserver ..." - - case $appserver in - tomcat) - # We do not want the Escenic jars to share the same - # classloader folder as tomcat does We thereby want - # clients to use a separate escenic classloader to avoid - # strange upgrade problems i.e wrong versions of certain - # libraries. - if [ -d $tomcat_base/escenic/lib ]; then - if [ `ls $tomcat_base/escenic/lib | grep .jar | wc -l` -gt 0 ]; then - rm $tomcat_base/escenic/lib/*.jar - exit_on_error "removing previous deployed libraries" - fi - cp $dir/lib/*.jar \ - $tomcat_base/escenic/lib \ - 1>>$log_file \ - 2>>$log_file - exit_on_error "deploying jar files to $tomcat_base/escenic/lib" - else - print "Could not find $tomcat_base/escenic/lib. Exiting." - print "Also make sure that you have defined this directory in" - print " $tomcat_base/conf/catalina.properties" - print "see sample configuration in the engine distribution" - print " contrib/appserver/tomcat/catalina-sample.properties" - exit 1 - fi - - exit_on_error "copying lib files to app server classpath" - - rm -rf $tomcat_base/work/* \ - 1>>$log_file \ - 2>>$log_file - - exit_on_error "removing work files from tomcat" - - for war in $dir/*.war ; do - if [ -d $tomcat_base/webapps/`basename $war .war` ] ; then - rm -rf $tomcat_base/webapps/`basename $war .war` \ - 1>>$log_file \ - 2>>$log_file - exit_on_error "removing already deployed escenic wars in $tomcat_base/webapps/" - fi - done - - # this scenario is likely when running many minimal - # instances of tomcat and some of these are not properly - # initialised. - if [ ! -d $tomcat_base/webapps ]; then - print $tomcat_base/webapps "doesn't exist, exiting." - exit 1 - fi - - if [ -n "$deploy_webapp_white_list" ]; then - deploy_this_war=0 - message="Deployment white list active, only deploying: " - message=$message"$deploy_webapp_white_list" - print $message - log $message - fi - - for war in $dir/*.war ; do - name=`basename $war .war` - - deploy_this_war=1 - if [ -n "$deploy_webapp_white_list" ]; then - deploy_this_war=0 - - for el in $deploy_webapp_white_list; do - if [ $el = $name ]; then - debug "found $war in white list, will deploy it" - deploy_this_war=1 - fi - done - fi - - if [ $deploy_this_war -eq 0 ]; then - continue - fi - - (cd $tomcat_base/webapps && - mkdir $name && - cd $name && - jar xf $war \ - 1>>$log_file \ - 2>>$log_file) - exit_on_error "extracting $war to $tomcat_base/webapps/" - done - ;; - - resin) - if [ ! -d $resin_home/deploy ]; then - mkdir -p $resin_home/deploy \ - 1>>$log_file \ - 2>>$log_file - fi - cp $ear $resin_home/deploy \ - 1>>$log_file \ - 2>>$log_file - ;; - *) - print "Deployment is only implemented for Resin and Tomcat so far." - ;; - esac -} - - -function start_type() -{ - unset CLASSPATH - message="Starting the $instance instance of $type on $HOSTNAME ..." - print $message - log $message - - if [ $type = "rmi-hub" ]; then - ensure_that_required_fields_are_set $hub_required_fields - if [ -r $rmi_hub_conf ]; then - export CLASSPATH=$rmi_hub_conf:$CLASSPATH - else - print $rmi_hub_conf "must point to a valid Nursery configuration" - print "for the rmi-hub, you may copy the one found in" - print "$ece_home/contrib/rmi-hub/config." - print "Exiting :-(" - exit 1 - fi - - for el in $rmi_hub_home/lib/*.jar; do - export CLASSPATH=$CLASSPATH:$el - done - - $java_home/bin/java \ - -Djava.rmi.server.hostname=${ece_server_hostname} \ - neo.nursery.GlobalBus /Initial \ - 1>>$log_file \ - 2>>$log_file & pid=$! - - echo $pid > $pid_file - exit 0 + verify_that_directory_and_file_are_writeable $log + verify_that_directory_and_file_are_writeable $pid_file - elif [ $type = "search" ]; then - # TODO trim & tun the default parameters for the search - # instance. - ensure_that_required_fields_are_set $engine_required_fields - elif [ $type = "engine" ]; then - ensure_that_required_fields_are_set $engine_required_fields - elif [ $type = "analysis" ]; then - ensure_that_required_fields_are_set $analysis_required_fields + if [ -z "$command" ]; then + print "You must specificy a command, see 'ece help'." + exit 1 fi - # indexer and engine are treated the same - case $appserver in - tomcat) - # Tomcat respects JAVA_OPTS set in configure(), so no need - # to set them here. - - if [ ! -x $tomcat_home/bin/catalina.sh ]; then - print "$tomcat_home/bin/catalina.sh was not executable" - print "unable to start tomcat" - exit 1 - fi - - # We call run here to get the log output to the stdout log - $tomcat_home/bin/catalina.sh run \ - 1>>$log_file \ - 2>>$log_file & pid=$! - ;; - oc4j) - export OC4J_JVM_ARGS=$ece_args - $oc4j_home/bin/oc4j -start\ - 1>>$log_file\ - 2>>$log_file & pid=$! - ;; - resin) - # Resin has stared insisting on a -J prefix of the -D - # prefixes :-) Tested with Resin 3.0.25 - resin_ece_args=`echo $ece_args | sed 's/-D/-J-D/g'` - - # works for Resin 3.0 - if [ -e $resin_home/bin/wrapper.pl ]; then - exec perl $resin_home/bin/wrapper.pl \ - -chdir \ - -name httpd \ - -class com.caucho.server.resin.Resin \ - $resin_ece_args ${1+"$@"} \ - 1>>$log_file \ - 2>>$log_file & pid=$! - else - # works for Resin 3.1 - $java_home/bin/java $ece_args \ - -jar $resin_home/lib/resin.jar \ - start \ - 1>>$log_file \ - 2>>$log_file & pid=$! - fi - ;; - jboss) - $jboss_home/bin/run.sh \ - -b 0.0.0.0 \ - -c $jboss_conf \ - 1>>$log_file \ - 2>>$log_file & pid=$! - ;; - *) - echo "" # extra line feed, because of the echo -n above - print "No appserver is defined in $ece_conf" + local instance_list=$(get_instance_list) + # checks if the user is using the default instance when he/she + # really wants to start one of the instances installed on the + # system. + if [ "$instance" == "default" -a \ + "$type" == "engine" -a \ + $(echo $command | egrep 'help' | wc -l) -eq 0 ]; then + if [[ -n "$instance_list" && \ + $(echo $instance_list | grep ' ' | wc -l) -gt 0 ]]; then + print "You must specify the instance with -i " + print "e.g. ece -i $(echo $instance_list | cut -d' ' -f1) $command" + print "Instances available on $HOSTNAME: $instance_list" exit 1 - esac - - if [ $pid ]; then - verify_that_directory_and_file_are_writeable $pid_file - echo $pid > $pid_file - fi - - exit_on_error $message -} - -function stop_type() -{ - message="Stopping the $instance instance of $type on $HOSTNAME ..." - - if [ -n "$type_pid" ]; then - log $message - print $message - - if [ -r $pid_file ]; then - if [ "$type_pid" != "`cat $pid_file`" ]; then - print "Is running, but was not started with `basename $0`" - print "system PID $ece_pid differs from the PID in $pid_file" - print "removing dangling PID file $pid_file. " - print "In future, be sure to use $0 to start " - print "and stop your $type" - kill $type_pid 1>>$log_file 2>>$log_file - rm $pid_file - return - fi - fi - - kill $type_pid \ - 1>>$log_file \ - 2>>$log_file - - if [ -e $pid_file ]; then - rm $pid_file \ - 1>>$log_file \ - 2>>$log_file fi - else - print "The $instance instance of $type on $HOSTNAME is NOT running" fi - - exit_on_error $message -} -function kill_type() -{ - if [ -n "$type_pid" ]; then - message="Using force to stop the $instance instance of $type on $HOSTNAME ..." - log $message - print $message - kill -9 $type_pid - if [ -w $pid_file ]; then - rm $pid_file - fi - else - print "No $instance instance of $type on $HOSTNAME to be killed" - fi - - exit_on_error "kill_type" -} + local instance_exists=0 + for el in $instance_list; do + if [ $el == $instance ]; then + instance_exists=1 + break + fi + done -function restart_type() -{ - stop_type - - # I've tested this with various values, 8 seconds seems to - # guarantee that the previous JVM process has stopped - # properly before starting the new one. - sleep 8 - - # sometimes the JVM refuses to shot down when doing a graceful - # kill, therefore, if it's still running after the sleep above, we - # use brute force to kill it. - set_type_pid - if [ -n "$type_pid" ]; then - message="The $instance instance of $type failed to stop gracefully" - debug $message - log $message >> $log_file - print $message - kill_type + if [ $instance_exists -eq 0 ]; then + print "Instance '$instance' doesn't exist on $HOSTNAME" + exit 1 fi - - start_type -} - -function assemble() -{ - if [ ! $type = "engine" ]; then - print "You cannot assemble $type" + + # verifies that java_home exists and has the java executable + if [ ! -d $java_home ]; then + print "java_home $java_home doesn't exist" + print "check your setting in one of: ${ece_conf_files_read[@]}" + exit 1 + elif [ ! -x $java_home/bin/java ]; then + print "$java_home/bin/java isn't executable for $ece_user" exit 1 fi - - message="Assembling your EAR file ..." - print $message - log $message >> $log_file - - cd $assemblytool_home && \ - ant -q ear -DskipRedundancyCheck=true \ - 1>>$log_file \ - 2>>$log_file - exit_on_error "$message" - - mkdir -p $ear_cache_dir/ - exit_on_error "creating $ear_cache_dir" - - cp $assemblytool_home/dist/engine.ear $cache_dir - exit_on_error "copying ear to $ear_cache_dir" - - debug $assemblytool_home/dist/engine.ear "is now ready" } -function tail_messages_log() -{ - for el in $log_file_list; do - if [ -r $el ]; then - print "tailing $el" - tail -f $el - break - fi - done +function run() { + $@ >> $log 2>> $log + exit_on_error "$@" } -function tail_out_log() -{ - tail_list=$log_file - - # if needs be, we can add more system out logs here. For now, - # we're sticking with the default one. - - print "Tailing the system out log $tail_list" - tail -f $tail_list -} +# Returns the file (can be a directory) passed to the function only +# if it's the actual file/directory and not a link to it. If the +# passed file is a link, the link target is returned instead. +# +# $1 - the file (which could be a link) +function get_actual_file() { + if [ -h ${1} ]; then + dir=$(dirname $1) + real_file=$(ls -l ${1} | awk '{print $11}') + + # Because of the test if the file we want to returns is + # absolute, we go to the root before testing. We want to + # preserve the cwd, therefore, we're doing the "cd /" in a + # subshell. + real_file=$( + cd / + if [ ! -e ${real_file} ]; then + real_file=${dir}/${real_file} + fi -function tail_app_log() -{ - if [ $type = "rmi-hub" ]; then - print "There is no application server log for $type" - exit 1 - fi - - if [ $appserver = "tomcat" ]; then - log_file=$tomcat_base/logs/localhost.`date +%F`.log - elif [ $appserver = "resin" -a -e $resin_home/log/jvm-default.log ]; then - log_file=$resin_home/log/jvm-default.log + # this is the return value from the sub process + echo ${real_file} + ) else - print "I don't know where the logs for $appserver are." - print "Ask support@escenic.com to add support for $appserver in " - print "tail_app_log()" - exit 1 + real_file=${1} fi - - print "Tailing the application server log $log_file" - tail -f $log_file + + echo ${real_file} } -function make_thread_dump() -{ +function set_type_command_and_instance() { + local next_is_type=0 + local next_is_instance=0 + local next_is_publication=0 + local next_is_resource=0 + local next_is_http_user=0 + local next_is_http_password=0 + local next_is_file=0 + + for el in $@; do + if [ "$el" == "-v" -o "$el" == "--verbose" ]; then + debug=1 + continue + fi - if [ -n "$type_pid" ]; then - print "Thread dump (PID" $type_pid") written to system out log." - print "Type 'ece -t $type -i default outlog' to see it or view" - print $log_file "directly." + if [ "$el" == "--full" ]; then + everything_but_the_kitchen_sink=1 + continue + fi - if [ -x $java_home/bin/jstack ]; then - jstack -l $type_pid >> $log_file - else - kill -QUIT $type_pid >> $log_file + if [ "$el" == "-f" -o $el == "--file" -o $el == "--uri" ]; then + next_is_file=1 + continue fi - else - get_status - fi -} -function set_type_command_and_instance() -{ - next_is_type=0 - next_is_instance=0 - next_is_publication=0 - next_is_resource=0 - - for el in $@; do - if [ $el = "-v" -o $el = "--verbose" ]; then - verbose=1 + if [ "$el" == "-q" -o $el == "--quiet" ]; then + quiet=1 continue fi - if [ $el = "--help" ]; then + + if [ "$el" == "--help" ]; then command=help continue fi - if [ $next_is_type -eq 1 ]; then + if [ "$next_is_file" -eq 1 ]; then + file=$el + next_is_file=0 + continue + fi + + if [ "$next_is_type" -eq 1 ]; then type=$el next_is_type=0 continue fi - - if [ $next_is_instance -eq 1 ]; then + + if [ "$next_is_http_user" -eq 1 ]; then + http_user=$el + next_is_http_user=0 + continue + fi + + if [ "$next_is_http_password" -eq 1 ]; then + http_password=$el + next_is_http_password=0 + continue + fi + + if [ "$next_is_instance" -eq 1 ]; then instance=$el next_is_instance=0 continue fi - - if [ $next_is_publication -eq 1 ]; then + + if [ "$next_is_publication" -eq 1 ]; then publication=$el next_is_publication=0 continue fi - - if [ $next_is_resource -eq 1 ]; then + + if [ "$next_is_resource" -eq 1 ]; then resource=$el next_is_resource=0 continue fi - - if [ $el = "-t" -o $el = "--type" ]; then + + if [ "$el" == "-t" -o "$el" == "--type" ]; then next_is_type=1 continue else next_is_type=0 fi - if [ $el = "-i" -o $el = "--instance" ]; then + if [ "$el" == "-i" -o "$el" == "--instance" ]; then next_is_instance=1 continue else next_is_instance=0 fi - if [ $el = "-p" -o $el = "--publication" ]; then + if [ "$el" == "-p" -o "$el" == "--publication" ]; then next_is_publication=1 continue else next_is_publication=0 fi - - if [ $el = "-r" -o $el = "--publication-resource" ]; then + + if [ "$el" == "-r" -o "$el" == "--publication-resource" ]; then next_is_resource=1 continue else next_is_resource=0 fi - + + if [ "$el" == "-u" -o "$el" == "--user" ]; then + next_is_http_user=1 + continue + else + next_is_http_user=0 + fi + + if [ "$el" == "-w" -o "$el" == "--password" ]; then + next_is_http_password=1 + continue + else + next_is_http_password=0 + fi + + if [ "$el" == "--exclude-binaries" ]; then + backup_exclude_binaries=1 + continue + elif [ "$el" == "--exclude-solr" ]; then + backup_exclude_solr=1 + continue + elif [ "$el" == "--exclude-conf" ]; then + backup_exclude_conf=1 + continue + elif [ "$el" == "--exclude-multimedia" ]; then + backup_exclude_multimedia=1 + continue + elif [ "$el" == "--exclude-db" ]; then + backup_exclude_db=1 + continue + elif [ "$el" == "--exclude-init" ]; then + backup_exclude_init=1 + continue + elif [ "$el" == "--exclude-state" ]; then + backup_exclude_state=1 + continue + elif [ "$el" == "--version" -o "$el" == "-V" ]; then + echo "Version:" $ece_scripts_version + exit 0 + fi + # the only thing left at this point, is the command command="$command $el" done -} - -function clean_up() -{ - if [ $type = "engine" ]; then - print "Cleaning up generated files in $assemblytool_home ..." - cd $assemblytool_home - ant clean \ - 1>>$log_file \ - 2>>$log_file - fi - tmp_dir_prefix="`basename $0`-" - if [ `ls $tmp_dir | grep $tmp_dir_prefix | wc -l` -gt 0 ]; then - print "Cleaning up generated files in $tmp_dir ..." - rm -rf $tmp_dir/$tmp_dir_prefix-[0-9]* \ - 1>>$log_file \ - 2>>$log_file - fi - - if [ -e /var/cache/escenic/ -a \ - `ls /var/cache/escenic | grep ece- | wc -l` -gt 0 ]; then - print "Cleaning up "`basename $0`" files in /var/cache/escenic/ ..." - rm -rf /var/cache/escenic/* \ - 1>>$log_file \ - 2>>$log_file + # If the user didn't specify which instance to use and if there's + # only one available instance, use that instead of asking the user + # to provide it in sanity_check. + if [ "$instance" == "default" ]; then + debug "Trying to determine instance name" + local instance_list=$(get_instance_list) + + if [ $(echo $instance_list | grep ' ' | wc -l) -eq 0 ]; then + instance=$instance_list + if [ -z "$instance" ] ; then + instance=default + fi + debug "setting instance=$instance as there's only one" + fi fi - } -function set_id() -{ - if [ $instance = "default" ]; then - id="["`basename $0`"#${type}]" +function set_id() { + if [ "$instance" == "default" ]; then + id="["$(basename $0)"#${type}]" else - id="["`basename $0`"#${type}-${instance}]" + id="["$(basename $0)"#${type}-${instance}]" fi - + debug type is $type \ and command is $command \ and instance is $instance } -function get_status() -{ +function get_status() { if [ -z "$type_pid" ]; then - print "DOWN" + echo "DOWN" exit 0 elif [ -r $pid_file ]; then - + if [ "$type_pid" != `cat $pid_file` ]; then - print "Is running, but was not started with `basename $0`" - print "system PID $ece_id differs from the PID in $pid_file" + echo "Is running, but was not started with $(basename $0)" + echo "system PID $ece_id differs from the PID in $pid_file" exit 1 fi else - print $pid_file "did not exist, " - print "the ${instance} instance of ${type} is running with PID $type_pid" - print "but hasn't been started properly with `basename $0`" + echo $pid_file "did not exist, " + echo "the ${instance} instance of ${type} is running with PID $type_pid" + echo "but hasn't been started properly with $(basename $0)" exit 1 fi - now=`date +%s` - started=`stat -c %Y $pid_file` - seconds=$(( now - started )) - days=$(( seconds / ( 60 * 60 * 24 ) )) - seconds_left=$(( seconds - ( $days * 60 * 60 * 24 ) )) - hours=$(( seconds_left / ( 60 * 60 ) )) - seconds_left=$(( seconds_left - ( $hours * 60 * 60 ) )) - minutes=$(( seconds_left / 60 )) - seconds_left=$(( seconds_left - $minutes * 60 )) - - print "UP" ${days}d ${hours}h ${minutes}m ${seconds_left}s + local now=`date +%s` + local started=`stat -c %Y $pid_file` + local seconds=$(( now - started )) + local days=$(( seconds / ( 60 * 60 * 24 ) )) + local seconds_left=$(( seconds - ( $days * 60 * 60 * 24 ) )) + local hours=$(( seconds_left / ( 60 * 60 ) )) + local seconds_left=$(( seconds_left - ( $hours * 60 * 60 ) )) + local minutes=$(( seconds_left / 60 )) + local seconds_left=$(( seconds_left - $minutes * 60 )) + + echo "UP" ${days}d ${hours}h ${minutes}m ${seconds_left}s } -function list_versions() -{ - if [ -z "$type_pid" ]; then - print "$instance instance of $type on $HOSTNAME is NOT running" - exit 1 +function read_rc_file_if_present() { + local rc_file=$HOME/.ecerc + if [ -r $rc_file ]; then + source $rc_file fi - - version_manager=escenic-admin/browser/Global/neo/io/managers/VersionManager - url=http://$host:$port/$version_manager - - print "Installed on the ${type} running on ${host}:${port} is:" - wget -O - $url 2>/dev/null | \ - grep "\[\[" | \ - sed 's/\[//g' | \ - sed 's/\]//g' | \ - sed 's/Name\=io/Name\=content-engine/g' | \ - sed 's/Name\=//g' | \ - sed 's/\;\ Version\=/\ /g' | \ - awk -F',' '{for (i = 1; i <= NF; i++) print " * " $i;}' | \ - # This is a hack since that for some reason, cannot get - # sub(/^[ \t]+/, "") to work inside the loop for $i, it seems - # to always operate on the incoming line. - sed 's/\*\ \ \ /\*/g' | \ - sort } -function update_publication_resources() -{ - url=http://${host}:${port}/escenic-admin/publication-resources +function get_escenic_admin_url() { + set_type_port + local url=http://${host}:${port}/escenic-admin + echo ${url} +} - if [ ! -r $resource ]; then - print $resource "doesn't exist. I will exit :-(" - exit 1 - fi +function show_all_publications() { + set_type_port + print "These are all the publications on ${instance}:" + get_publication_list ${port} +} - if [ -x $publication ]; then - print "You must specify which publication to update (-p )" - exit 1 - fi - - case "$(basename $resource)" in - content-type) - url=${url}/${publication}/escenic/content-type - ;; - feature) - url=${url}/${publication}/escenic/feature - ;; - layout) - url=${url}/${publication}/escenic/layout - ;; - layout-group) - url=${url}/${publication}/escenic/layout-group - ;; - image-version) - url=${url}/${publication}/escenic/image-version - ;; - menu) - url=${url}/${publication}/escenic/plugin/menu - ;; - - *) - print "Invalid resource: $(basename $resource) :-(" - exit 1 - - esac - - print "Updating the $(basename $resource) resource for the $publication" \ - "publication" - - debug POSTing $resource to $url - wget -O - \ - --post-file ${resource} \ - $url \ - 1>>$log_file 2>>$log_file - +function show_all_deployments() { + print "These are all the publications on ${instance}:" + cat $(get_deployment_log) } +init +read_rc_file_if_present set_type_command_and_instance $@ set_id set_common_settings @@ -1016,61 +793,13 @@ set_type_settings set_instance_settings sanity_check -function print_help() -{ - echo "Usage: $0 [-t ] [-i ] " - echo "" - echo "DESCRIPTION" - echo " -t --type " - echo " The following types are available:" - echo " engine - The Escenic Content Engine, this is the default" - echo " and is the assumed type if none is specified." - echo " search - A standalone search indexer and solr instance" - echo " rmi-hub - The RMI hub responsible for the internal " - echo " communication between the ECE instances." - echo " analysis - The Escenic Analysis Engine also knows as 'Stats'" - echo "" - echo " -i --instance " - echo " The type instance, such as editor1, web1 or search1" - echo "" - echo " -p --publication " - echo " Needed only for updating publication resources" - echo "" - echo " -r --resource " - echo " Used for updating publication resources." - echo " Must be one of: content-type, feature, layout, layout-group" - echo " image-version, menu" - echo "" - echo " -v --verbose" - echo " Prints out debug statements, useful for debugging." - echo "" - echo "The following commands are available:" - echo " applog the type's app server log" - echo " assemble runs the Assembly Tool *)" - echo " clean removes temporary files created by $0 *)" - echo " deploy deploys the assembled EAR *)" - echo " help prints this help screen" - echo " kill uses force to stop the type" - echo " log the type's Log4J log" - echo " outlog the $id script log (system out log)" - echo " restart restarts the type" - echo " start starts the type" - echo " status checks if the type is running" - echo " stop stops the type" - echo " threaddump write a thread dump to standard out (system out log)" - echo " update update publication resources" - echo " versions lists the ECE component versions" - echo "" - echo "*) only applicable if type is 'engine'" -} - for el in $command; do case "$el" in start) start_type ;; status) - get_status + print "$(get_status)" ;; stop) stop_type @@ -1078,9 +807,21 @@ for el in $command; do restart) restart_type ;; + info) + get_info_for_type + ;; log) tail_messages_log ;; + list-logs) + show_all_log_paths + ;; + list-publications) + show_all_publications + ;; + list-deployments) + show_all_deployments + ;; outlog) tail_out_log ;; @@ -1099,24 +840,38 @@ for el in $command; do assemble) assemble ;; + package) + create_packages + ;; clean) clean_up ;; versions) list_versions ;; + list-instances) + print $(get_instance_list) + ;; + remove-old-log-files) + remove_old_log_files + ;; update) update_publication_resources ;; + edit) + update_publication_resources "edit_first" + ;; + backup) + backup_type + ;; + flush) + flush_caches + ;; + top) + run_ece_top + ;; help) - if [ -x $(which less) ]; then - print_help | less - elif [ -x $(which more) ]; then - print_help | more - else - print_help - fi - + print_help ;; *) print "Invalid command: '$el' :-(" @@ -1126,4 +881,3 @@ for el in $command; do done exit 0 - diff --git a/usr/bin/ece-import b/usr/bin/ece-import new file mode 100755 index 00000000..e32e930b --- /dev/null +++ b/usr/bin/ece-import @@ -0,0 +1,418 @@ +#! /usr/bin/env bash + +## Runs one VOSA import job one time. The output is XML suitable for +## the standard Escenic Syndication XML import job. +## +## The script can also create an import job from an import job archive +## +## See /usr/share/doc/vizrt/vosa-handbook/import-jobs.org for more +## details on the structures this command operates on. + +function bootstrap_thyself() { + # first, try to be nice, then check the standard location + local dir=$(dirname $0)/../share/escenic/ece-scripts + if [ ! -d $dir ]; then + dir=/usr/share/escenic/ece-scripts + fi + + local common_libraries=" + common-bashing.sh + common-ece.sh + common-io.sh + common-os.sh + " + + for el in $common_libraries; do + source $dir/$el 2>/dev/null || { + echo "$(basename $0): Could not load the library $el," \ + "and I can't live without it :-(" | fmt + exit 1 + } + done + + for el in $dir/$(basename $0).d/*.sh; do + source $el 2>/dev/null || { + echo "$(basename $0): Could not load the library $el," \ + "and I can't live without it :-(" | fmt + exit 1 + } + done +} + +bootstrap_thyself + +# internal variables +escenic_group=escenic +escenic_spool_base_dir=/var/spool/escenic/import +escenic_user=escenic +job_name="" +log_base_dir=/var/log/escenic +ece_scripts_version="straight-from-github" + +log=$log_base_dir/$(basename $0).log +nursery_base_dir=/etc/escenic/engine/common +publication_name="" +raw_spool_base_dir=/var/spool/escenic/raw +raw_state_base_dir=/var/lib/escenic/raw +raw_transformation_base_dir=/var/cache/escenic/import +raw_transformed_base_dir=/var/backups/escenic/import +transformers_base_dir=/usr/share/escenic/import +arg_regex_of_file="^(.*)$" +arg_write_url="" + +# available commands/operations for ece-import +COMMAND_IMPORT=1 +COMMAND_CREATE_IMPORT_CONFIGURATION=2 +COMMAND_DOWNLOAD_RAW_DATA=3 +command=$COMMAND_IMPORT + +function get_user_input() { + local next_is_name=0 + local next_is_publication=0 + local next_is_import_archive=0 + local next_is_nursery_base_dir=0 + local next_is_escenic_user=0 + local next_is_escenic_group=0 + local next_is_user=0 + local next_is_password=0 + local next_is_uri=0 + local next_is_http_proxy=0 + local next_is_regex_of_file=0 + local next_is_write_url=0 + + for el in $@; do + if [[ "$el" == "-n" || "$el" == "--name" ]]; then + next_is_name=1 + elif [[ "$el" == "-p" || "$el" == "--publication" ]]; then + next_is_publication=1 + elif [[ "$el" == "-f" || "$el" == "--import-archive" ]]; then + next_is_import_archive=1 + elif [[ "$el" == "--escenic-user" ]]; then + next_is_escenic_user=1 + elif [[ "$el" == "--escenic-group" ]]; then + next_is_escenic_group=1 + elif [[ "$el" == "--nursery-base-dir" ]]; then + next_is_nursery_base_dir=1 + elif [[ "$el" == "--user" ]]; then + next_is_user=1 + elif [[ "$el" == "--password" ]]; then + next_is_password=1 + elif [[ "$el" == "--uri" ]]; then + next_is_uri=1 + elif [[ "$el" == "--regex-of-file" ]]; then + next_is_regex=1 + elif [[ "$el" == "--write_url" ]]; then + next_is_write_url=1 + elif [[ "$el" == "--directories-only" ]]; then + directories_only=1 + elif [[ "$el" == "--http-proxy" ]]; then + next_is_http_proxy=1 + elif [ "$el" == "--version" -o "$el" == "-V" ]; then + echo "Version:" $ece_scripts_version + exit 0 + elif [ $next_is_name -eq 1 ]; then + job_name=$el + next_is_name=0 + elif [ $next_is_publication -eq 1 ]; then + publication_name=$el + next_is_publication=0 + elif [ $next_is_import_archive -eq 1 ]; then + import_archive=$el + next_is_import_archive=0 + elif [ $next_is_nursery_base_dir -eq 1 ]; then + nursery_base_dir=$el + next_is_nursery_base_dir=0 + elif [ $next_is_escenic_user -eq 1 ]; then + escenic_user=$el + next_is_escenic_user=0 + elif [ $next_is_escenic_group -eq 1 ]; then + escenic_group=$el + next_is_escenic_group=0 + elif [ $next_is_user -eq 1 ]; then + user=$el + next_is_user=0 + elif [ $next_is_password -eq 1 ]; then + password=$el + next_is_password=0 + elif [ $next_is_uri -eq 1 ]; then + uri=$el + next_is_uri=0 + elif [ $next_is_regex_of_file -eq 1 ]; then + arg_regex_of_file=$el + next_is_regex_of_file=0 + elif [ $next_is_write_url -eq 1 ]; then + arg_write_url=$el + next_is_write_url=0 + elif [ $next_is_http_proxy -eq 1 ]; then + the_http_proxy="$el" + next_is_http_proxy=0 + else + if [[ "$el" == "create" ]]; then + command=$COMMAND_CREATE_IMPORT_CONFIGURATION + elif [[ "$el" == "download-import-data" ]]; then + command=$COMMAND_DOWNLOAD_RAW_DATA + fi + fi + done + + local errors=0 + if [ -z "$job_name" -a -z "${import_archive}" ]; then + print_and_log "You must specify which import job to run" + print_and_log "E.g.: $(basename $0) --name video" + errors=1 + fi + if [ -z "$publication_name" -a -z "${import_archive}" ]; then + print_and_log "You must specify the publication name" + print_and_log "E.g.: $(basename $0) --publication mypub" + errors=1 + fi + + if [ -n "${import_archive}" -a ! -r "${import_archive}" ]; then + print_and_log "You have specified an import job archive file" \ + "but it doesn't exist :-(" + errors=1 + fi + + if [ $command -eq $COMMAND_DOWNLOAD_RAW_DATA ]; then + if [ -z "$user" ]; then + print_and_log "You must specify the the user" + print_and_log "E.g.: $(basename $0) --user lisa" + errors=1 + fi + if [ -z "$password" ]; then + print_and_log "You must specify the the user" + print_and_log "E.g.: $(basename $0) --password foo" + errors=1 + fi + if [ -z "$uri" ]; then + print_and_log "You must specify the the URI" + print_and_log "E.g.: $(basename $0) --uri http://feeds.com/myfeed" + errors=1 + fi + fi + + if [ $errors -eq 1 ]; then + remove_pid_and_exit_in_error + fi +} + +## $1 :: the transformer (file name, relative or absoulte) +function is_transformer_supported() { + if [ -z $1 ]; then + return + fi + + local supported_transformer_list="pl py sh xsl" + for el in $supported_transformer_list; do + if [[ "$1" == *"${el}" ]]; then + echo 1 + return + fi + done + + echo 0 +} + +## $1 :: file +function perform_transformations() { + for el in $transformers_base_dir/$publication_name/$job_name/transformers/[0-9]*; do + if [ $(is_transformer_supported $el) -eq 0 ]; then + log "$(yellow WARNING) The transformer $el isn't supported by $(basename $0)" + continue + fi + + log "Applying transformation $(basename $el) to $1" + + if [[ "$el" == *".sh" ]]; then + bash $el $1 >> $log 2>> $log + + if [ $? -gt 0 ]; then + handle_transformation_error $el $1 + return + fi + elif [[ "$el" == *".xsl" ]]; then + xsltproc --output ${1}.tmp ${el} ${1} >> $log 2>> $log + + if [ $? -gt 0 ]; then + handle_transformation_error $el $1 + return + else + run mv ${1}.tmp ${1} + fi + elif [[ "$el" == *".pl" ]]; then + perl $el $1 >> $log 2>> $log + if [ $? -gt 0 ]; then + handle_transformation_error $el $1 + return + fi + elif [[ "$el" == *".py" ]]; then + python $el $1 >> $log 2>> $log + if [ $? -gt 0 ]; then + handle_transformation_error $el $1 + return + fi + fi + transformation_count=$(( transformation_count + 1 )) + done +} + +## Will log the transformer error and move it to the error archive. +## +## $1 :: transformer +## $2 :: the raw/input file +function handle_transformation_error() { + local dir=$raw_transformed_base_dir/$publication_name/$job_name/failed + log "$(red FAILED) The transformation $1 on file $2" \ + "moving $2 to $dir and skipping to the next XML file" + run mv $2 $dir +} + +import_error_count=0 + +## $1 :: the directory to check for multimedia files. +## $2 :: the directory to move any of these multimedia files to +function move_any_multimedia_files_if_present() { + if [ ! -d $1 -o ! -d $2 ]; then + return + fi + + local multimedia_file_count=$( + ls $1 | egrep -i ".(png|gif|jpg|jpeg|pdf)$" | wc -l + ) + if [ $multimedia_file_count -gt 0 ]; then + log "Moving ${multimedia_file_count} multimedia files from $1 to $2" + mv $1/*.{png,gif,jpg,jpeg,pdf} $2 >> $log 2>/dev/null + fi +} + +function import_raw_files() { + raw_file_count=0 + for f in $(find $raw_spool_base_dir/$publication_name/$job_name -type f); do + raw_file_count=$(( raw_file_count + 1 )) + transformation_count=0 + print_and_log "Importing raw XML #${raw_file_count}: $(basename $f) ..." + local file=$raw_transformation_base_dir/$publication_name/$job_name/$(basename $f) + run cp $f $file + perform_transformations $file + + log "Applied $transformation_count transformations to $file" + if [ $(is_escenic_xml_ok $file) -eq 1 ]; then + local dir=$escenic_spool_base_dir/$publication_name/$job_name/new + log "Transformed XML is OK, moving transformed file to" $dir + move_any_multimedia_files_if_present $(dirname $file) $dir + run mv $file $dir + dir=$raw_transformed_base_dir/$publication_name/$job_name/succeeded + log "Transformed XML is OK, moving original raw XML to" $dir \ + "and gzip-ing it." + run mv $f $dir + run gzip --force $dir/$(basename $f) + else + local dir=$raw_transformed_base_dir/$publication_name/$job_name/failed + log $(red ERROR) "Transformed XML #${raw_file_count}," \ + $file "isn't valid Escenic Syndication XML, so moving it to" $dir + run mv $f $dir + import_error_count=$(( import_error_count + 1 )) + fi + done +} + +function verify_import_job_configuration() { + verify_writable_dir_list \ + $raw_spool_base_dir/$publication_name/$job_name \ + $raw_state_base_dir/$publication_name/$job_name \ + $raw_transformation_base_dir/$publication_name/$job_name \ + $raw_transformed_base_dir/$publication_name/$job_name \ + $escenic_spool_base_dir/$publication_name/$job_name/new \ + $escenic_spool_base_dir/$publication_name/$job_name/archive \ + $escenic_spool_base_dir/$publication_name/$job_name/error + verify_readable_dir_list $transformers_base_dir/$publication_name/$job_name + + local dir=$transformers_base_dir/$publication_name/$job_name/transformers + local tranformation_count=$( + ls $dir | \ + grep ^[0-9] | \ + egrep ".sh$|.pl$|.py$|.xsl$" | \ + wc -l + ) + + if [ $command -eq $COMMAND_IMPORT -a $tranformation_count -lt 1 ]; then + print_and_log "$(yellow WARNING) No transformers found in" \ + "$dir/, I'm assuming the incoming" \ + "data is already tranformed into Escenic Syndication XML" + fi + + print_and_log "Running import" $job_name \ + "for publication" $publication_name +} + +function print_report() { + if [ $command -eq $COMMAND_IMPORT ]; then + print_and_log "Number of raw XML files processed:" $raw_file_count + print_and_log "Number of raw XML successes:" \ + $(green $(( raw_file_count - import_error_count ))) + print_and_log "Number of raw XML errors:" $(red $import_error_count) + fi +} + +assert_commands_available xsltproc xmllint xml_grep +get_user_input $@ + +function run_import() { + pid_file=${pid_file/%.pid/-run-import.pid} + lock_file=${lock_file/%.lock/-run-import.lock} + common_pre_run + + verify_import_job_configuration + import_raw_files + common_post_run +} + +function run_create_import_configuration() { + pid_file=${pid_file/%.pid/-create.pid} + lock_file=${lock_file/%.lock/-create.lock} + common_pre_run + + if [ -z $import_archive ]; then + if [ ${directories_only-0} -eq 0 ]; then + create_import_configuration $publication_name $job_name + fi + create_import_directories $publication_name $job_name + else + apply_import_archive + fi + + common_post_run +} + +function run_download_raw_data() { + pid_file=${pid_file/%.pid/-download.pid} + lock_file=${lock_file/%.lock/-download.lock} + + common_pre_run + verify_import_job_configuration + download_latest_files + common_post_run +} + +function common_pre_run() { + print_and_log "Started @ $(date), I'm logging to $log" + create_pid + create_lock_or_fail +} + +function common_post_run() { + print_manual_steps + print_report + + print_and_log "Finished @ $(date), enjoy thyself!" + remove_pid + remove_lock +} + +if [[ "$command" == $COMMAND_IMPORT ]]; then + run_import +elif [[ "$command" == $COMMAND_CREATE_IMPORT_CONFIGURATION ]]; then + run_create_import_configuration +elif [[ "$command" == $COMMAND_DOWNLOAD_RAW_DATA ]]; then + run_download_raw_data +fi diff --git a/usr/bin/generate-changelog b/usr/bin/generate-changelog new file mode 100755 index 00000000..99fc980e --- /dev/null +++ b/usr/bin/generate-changelog @@ -0,0 +1,207 @@ +#! /usr/bin/env bash + +log=$HOME/.$(basename $0).log +archive_base_dir=$HOME/.$(basename $0) + +function bootstrap_thyself() { + # first, try to be nice, then check the standard location + local dir=$(dirname $0)/../share/escenic/ece-scripts + if [ ! -d $dir ]; then + dir=/usr/share/escenic/ece-scripts + fi + + local common_libraries=" + common-bashing.sh + common-io.sh + " + + for el in $common_libraries; do + source $dir/$el 2>/dev/null || { + echo "$(basename $0): Could not load the library $el," \ + "and I can't live without it :-(" | fmt + exit 1 + } + done +} + +function read_user_settings() { + local file=$HOME/.$(basename $0).conf + if [ -r $file ]; then + run source $file + else + log $(blue INFO) $file "doesn't exist, will default to Atlassian on Demand" + fi +} + +function get_header_from_jira() { + local body=$(curl -u ${user}:${password} -s ${jira_base_url}/browse/${1}) + echo "$body" | grep '' | sed -e 's/<title>//g' -e 's/<\/title>//g' +} + +function get_to_revision() { + echo ${to-COMMITTED} +} + +function get_from_revision() { + echo ${from-PREV} +} + +function get_user_input() { + local next_is_from=0 + local next_is_to=0 + local next_is_project=0 + local next_is_user=0 + local next_is_password=0 + + for el in "$@"; do + if [[ "$el" == "-s" || "$el" == "--from" ]]; then + next_is_from=1 + elif [[ "$el" == "-u" || "$el" == "--user" ]]; then + next_is_user=1 + elif [[ "$el" == "-p" || "$el" == "--password" ]]; then + next_is_password=1 + elif [[ "$el" == "-t" || "$el" == "--to" ]]; then + next_is_to=1 + elif [[ "$el" == "-p" || "$el" == "--project" ]]; then + next_is_project=1 + elif [[ "$el" == "-f" || "$el" == "--full" ]]; then + full_listing=1 + elif [ ${next_is_from-0} -eq 1 ]; then + from=$el + next_is_from=0 + elif [ ${next_is_user-0} -eq 1 ]; then + user=$el + next_is_user=0 + elif [ ${next_is_password-0} -eq 1 ]; then + password=$el + next_is_password=0 + elif [ ${next_is_to-0} -eq 1 ]; then + to=$el + next_is_to=0 + elif [ ${next_is_project-0} -eq 1 ]; then + project_code=$el + next_is_project=0 + fi + done +} + +function should_regenerate() { + if [[ $(is_number $(get_from_revision)) -eq 1 && \ + $(is_number $(get_to_revision)) ]]; then + echo 0 + return + fi + + echo 1 +} + +function get_commit_information_from_vcs() { + the_diff=$(get_archive_dir)/from-$(get_from_revision)-to-$(get_to_revision).diff + + if [[ ! -e $the_diff || $(should_regenerate) -eq 1 ]]; then + svn diff -r $(get_from_revision):$(get_to_revision) > $the_diff + exit_on_error "svn diff -r $(get_from_revision):$(get_to_revision)" + fi + + if [ ${full_listing-0} -eq 1 ]; then + cat $the_diff + else + echo "Full diff of all" $(egrep '^(\+|\-)' $the_diff | wc -l) \ + "changes:" $the_diff | fmt + fi +} + +## $@ :: svn revision number or tag name +function get_date_from_svn_log() { + echo $(svn log -r "$@" | sed -n '2p' | cut -d'|' -f3) +} + +function get_svn_location() { + svn info | grep URL | cut -d':' -f2- +} + +function get_info_from_jira() { + the_report=$(get_archive_dir)/from-$(get_from_revision)-to-$(get_to_revision).report + + if [[ ! -e $the_report || $(should_regenerate) -eq 1 ]]; then + echo "Changes in $(get_svn_location)" > $the_report + echo "From: revision $(get_from_revision) @" \ + $(get_date_from_svn_log $(get_from_revision)) >> $the_report + echo "To : revision $(get_to_revision) @" \ + $(get_date_from_svn_log $(get_to_revision)) >> $the_report + + local commit_log=$(svn log -r $(get_from_revision):$(get_to_revision)) + echo "$commit_log" | \ + grep ${project_code}-[0-9]* | \ + sed "s#.*\(${project_code}-[0-9]*\).*#\1#g" | \ + sort | \ + uniq | while read f; do + echo " *" $(get_header_from_jira $f) | fmt >> $the_report + echo " URL: ${jira_base_url}/browse/$(basename $f)" >> $the_report + echo "" >> $the_report + done + + add_risk_assemsment_to_report + fi + + echo "Report:" $the_report +} + +function sanity_check() { + if [ ! -e $(pwd)/.svn ]; then + print "This directory, $(pwd), " \ + "does not contain a working version control checkout." + exit 1 + fi + + # defaulting to Atlassian on demand + jira_base_url=${jira_base_url-https://vizrtcustomers.jira.com} + if [ -z "$svn_base_url" ]; then + # bash default string substitution terminates with slashes, hence + # have to set this manually here. + svn_base_url=${jira_base_url}/svn + fi + + conf_file=$HOME/.$(basename $0).conf + ensure_variable_is_set \ + project_code \ + user \ + password \ + jira_base_url \ + svn_base_url + + if [[ "$(get_from_revision)" == "$(get_to_revision)" ]]; then + print_and_log "From and to revision are the same," \ + "will no create any change log" + exit 0 + fi +} + +function get_project() { + echo $(lowercase ${project_code}) +} + +function get_archive_dir() { + local project_context=$(lowercase $(get_svn_location | \ + sed -e "s#${svn_base_url}/##g" -e 's#[ ]*##g') + ) + + local dir=${archive_base_dir}/${project_context} + make_dir $dir + echo $dir +} + +function add_risk_assemsment_to_report() { + echo "Risk assessment score: " \ + $(wc -l $the_diff 2>/dev/null | cut -d' ' -f1) \ + >> $the_report +} + +bootstrap_thyself +read_user_settings +get_user_input "$@" +sanity_check +get_commit_information_from_vcs +get_info_from_jira + + diff --git a/usr/bin/generate-git-changelog b/usr/bin/generate-git-changelog new file mode 100755 index 00000000..48dabf38 --- /dev/null +++ b/usr/bin/generate-git-changelog @@ -0,0 +1,194 @@ +#! /usr/bin/env bash + +log=$HOME/.$(basename $0).log +archive_base_dir=$HOME/.$(basename $0) + +function bootstrap_thyself() { + # first, try to be nice, then check the standard location + local dir=$(dirname $0)/../share/escenic/ece-scripts + if [ ! -d $dir ]; then + dir=/usr/share/escenic/ece-scripts + fi + + local common_libraries=" + common-bashing.sh + common-io.sh + " + + for el in $common_libraries; do + source $dir/$el 2>/dev/null || { + echo "$(basename $0): Could not load the library $el," \ + "and I can't live without it :-(" | fmt + exit 1 + } + done +} + +function get_header_from_jira() { + local body=$(curl -u ${user}:${password} -s ${jira_base_url}/browse/${1}) + echo "$body" | grep '<title>' | sed -e 's/<title>//g' -e 's/<\/title>//g' +} + +# --to revision number +function get_to_revision() { + echo ${to-COMMITTED} +} + +# --from revision number +function get_from_revision() { + echo ${from-PREV} +} + +function get_user_input() { + local next_is_from=0 + local next_is_to=0 + local next_is_project=0 + local next_is_user=0 + local next_is_password=0 + local next_is_jirabaseurl=0 + + for el in "$@"; do + if [[ "$el" == "-s" || "$el" == "--from" ]]; then + next_is_from=1 + elif [[ "$el" == "-u" || "$el" == "--user" ]]; then + next_is_user=1 + elif [[ "$el" == "-p" || "$el" == "--password" ]]; then + next_is_password=1 + elif [[ "$el" == "-t" || "$el" == "--to" ]]; then + next_is_to=1 + elif [[ "$el" == "-p" || "$el" == "--project" ]]; then + next_is_project=1 + elif [[ "$el" == "-j" || "$el" == "--jirabaseurl" ]]; then + next_is_jirabaseurl=1 + elif [[ "$el" == "-f" || "$el" == "--full" ]]; then + full_listing=1 + elif [ ${next_is_from-0} -eq 1 ]; then + from=$el + next_is_from=0 + elif [ ${next_is_user-0} -eq 1 ]; then + user=$el + next_is_user=0 + elif [ ${next_is_password-0} -eq 1 ]; then + password=$el + next_is_password=0 + elif [ ${next_is_to-0} -eq 1 ]; then + to=$el + next_is_to=0 + elif [ ${next_is_project-0} -eq 1 ]; then + project_code=$el + next_is_project=0 + elif [ ${next_is_jirabaseurl-0} -eq 1 ]; then + jirabaseurl=$el + next_is_jirabaseurl=0 + fi + done +} + + +function get_commit_information_from_vcs() { + the_diff=$(get_archive_dir)/from-$(get_from_revision)-to-$(get_to_revision).diff + + if [[ ! -e $the_diff ]]; then + git diff $(get_from_revision) $(get_to_revision) > $the_diff + exit_on_error "git diff $(get_from_revision) $(get_to_revision)" + fi + + if [ ${full_listing-0} -eq 1 ]; then + cat $the_diff + else + echo "Full diff of all" $(egrep '^(\+|\-)' $the_diff | wc -l) \ + "changes:" $the_diff | fmt + fi + + if [[ ! -z $jirabaseurl ]]; then + get_info_from_jira + fi + +} + +## $@ :: git last revision date +#Fri Sep 27 11:18:05 2013 +0600 +function get_date_from_git_log() { + echo $(git log -1 --format="%cd") +} + +# e.g. ssh://git@git.vizrtsaas.com/ccipoc +function get_git_location() { + echo $(git config --get remote.origin.url) +} + + +function get_info_from_jira() { + the_report=$(get_archive_dir)/from-$(get_from_revision)-to-$(get_to_revision).report + + if [[ ! -e $the_report ]]; then + echo "Changes in branch:$(get_project_branch) and location:$(get_git_location) " > $the_report + echo "From: revision $(get_from_revision) @" \ + $(get_date_from_git_log $(get_from_revision)) >> $the_report + echo "To : revision $(get_to_revision) @" \ + $(get_date_from_git_log $(get_to_revision)) >> $the_report + + local commit_log=$(git log $(get_from_revision)..$(get_to_revision)) + echo "$commit_log" | \ + grep $(get_jira_project_name)-[0-9]* | \ + sed "s#.*\($(get_jira_project_name)-[0-9]*\).*#\1#g" | \ + sort | \ + uniq | while read f; do + echo " *" $(get_header_from_jira $f) | fmt >> $the_report + echo " URL: ${jira_base_url}/browse/$(basename $f)" >> $the_report + echo "" >> $the_report + done + + add_risk_assemsment_to_report + fi + + echo "Report:" $the_report +} + +function sanity_check() { + #if [ ! -e $(pwd)/src/.git ]; then + # print "This directory, $(pwd), " \ + # "does not contain a working version control checkout." + #exit 1 + #fi + + # defaulting to Atlassian on demand + jira_base_url=${jirabaseurl} + + if [[ "$(get_from_revision)" == "$(get_to_revision)" ]]; then + print_and_log "From and to revision are the same," \ + "will no create any change log" + exit 0 + fi +} + +#e.g. ccipoc.git +function get_project_name() { + echo ${project_code} | awk '{split($0,array,"/")} END{print array[1]}' +} + +function get_jira_project_name() { + echo ${project_code} | awk '{split($0,array,"/")} END{print array[1]}' | sed 's/.git//g' | tr [a-z] [A-Z] + +} + +function get_project_branch() { + echo ${project_code} | awk '{split($0,array,"/")} END{print array[2]}' +} + +function get_archive_dir() { + local directory=${archive_base_dir}/$(get_project_name)/$(get_project_branch) + make_dir $directory + echo $directory +} + +function add_risk_assemsment_to_report() { + echo "Risk assessment score: " \ + $(wc -l $the_diff 2>/dev/null | cut -d' ' -f1) \ + >> $the_report +} + +bootstrap_thyself +get_user_input "$@" +sanity_check +get_commit_information_from_vcs diff --git a/usr/bin/sync-network-drive b/usr/bin/sync-network-drive new file mode 100755 index 00000000..6cec78cf --- /dev/null +++ b/usr/bin/sync-network-drive @@ -0,0 +1,198 @@ +#! /usr/bin/env bash + +# Script made for syncing the NFS exports between the serving +# hosts. The script ensures that there is only one instance of it +# running and thus that two simultaneous runs it will not cause +# corruption of data. +# +# +# Example usage with source (-s) and target (-t): +# +# $ sync-network-drive \ +# -s remote-server:/var/lib \ +# -t /var/backups/remote-server +# +# The full path on the remote server is replicated locally here, so +# after this command, you'll have remote-server's /var/lib directory +# under your local directory /var/backups/remote-server/var/lib And +# one last thing: trailing slashes are not needed + +log=/var/log/escenic/$(basename $0).log + +function bootstrap_thyself() { + # first, try to be nice, then check the standard location + local dir=$(dirname $0)/../share/escenic/ece-scripts + if [ ! -d $dir ]; then + dir=/usr/share/escenic/ece-scripts + fi + + for el in common-bashing.sh common-io.sh; do + source $dir/$el 2>/dev/null || { + echo "$(basename $0): Could not load the library $el," \ + "and I can't live without it :-(" | fmt + exit 1 + } + done +} + +bootstrap_thyself + +tmp_known_hosts_file=$(mktemp) + +rsync_opts=" +--recursive +--links +--perms +--group +--owner +--compress +--itemize-changes +--ignore-times +--checksum +--cvs-exclude +" +rsync_additional_outgoing_opts=" +--dry-run +--verbose +" +# the backslashes need to be here +rsync_ssh_opts="\ +-o BatchMode=yes \ +-o UserKnownHostsFile=${tmp_known_hosts_file} \ +-o StrictHostKeyChecking=no \ +-p 22 \ +" + +outgoing=0 +verbose=0 + +function ensure_src_is_sane() { + # checking that the src is <host>:<dir> + if [[ $(dirname $(echo "$src" | cut -d':' -f2 )) == /** ]]; then + return + else + print "Source $src is insane, should be <host>:<absolute dir>" + remove_lock + tidy_up + exit 1 + fi +} + +function ensure_target_dir_is_there() { + mkdir -p $target_dir || (echo "failed creating target dir"; exit 1 ) + + if [ ! -w $target_dir ]; then + print $USER "cannot write to $target_dir :-(" + remove_lock + tidy_up + exit 1 + fi +} + +function sync_it() { + if [ $outgoing -eq 1 ]; then + rsync_opts=${rsync_opts}" "${rsync_additional_outgoing_opts} + fi + + if [ $verbose -eq 1 ]; then + rsync_opts=${rsync_opts}" --verbose " + fi + + print "Sync on $HOSTNAME started @ $(stat -c %y $lock_file)" \ + "${src} -> ${target_dir}" + run rsync $rsync_opts \ + -e "ssh $rsync_ssh_opts" \ + $src/ \ + $target_dir/ +} + +function tidy_up() { + run rm $tmp_known_hosts_file +} + +function print_report() { + local now=$(date +%s) + local started=$(stat -c %Y $lock_file) + local seconds=$(( now - started )) + local days=$(( seconds / ( 60 * 60 * 24 ) )) + local seconds_left=$(( seconds - ( $days * 60 * 60 * 24 ) )) + local hours=$(( seconds_left / ( 60 * 60 ) )) + local seconds_left=$(( seconds_left - ( $hours * 60 * 60 ) )) + local minutes=$(( seconds_left / 60 )) + local seconds_left=$(( seconds_left - $minutes * 60 )) + + print "Hi! $src has now been synced to $target_dir" + print "It took" ${days}d ${hours}h ${minutes}m ${seconds_left}s + +} + +function get_user_options() { + while getopts ":s:t:ov" opt; do + case $opt in + s) + src=${OPTARG} + ;; + t) + target_dir_root=${OPTARG} + + if [[ $target_dir_root == "/" ]]; then + target_dir_root="" + fi + + target_dir=${target_dir_root}$(echo $src | cut -d":" -f2) + ;; + o) + outgoing=1 + ;; + v) + verbose=1 + ;; + \?) + print "Invalid option: -$OPTARG" >&2 + exit 1 + ;; + :) + print "Option -$OPTARG requires an argument." >&2 + exit 1 + ;; + *) + print "Command is $OPTARG" + exit 0 + esac + done +} + +function ensure_user_options_are_ok() { + if [ -z "${src}" ]; then + cat <<EOF +You must specify a source with -s <host>:<dir>, e.g.: +$(basename $0) -s app1:/var/log -t /var/backups/app1 +EOF + +tidy_up +exit 1 + fi + + if [ -z "${target_dir}" ]; then + cat <<EOF +You must specify a local target directorywith -t <target dir>, e.g.: +$(basename $0) -s app1:/var/log -t /var/backups/app1 +EOF + +tidy_up +exit 1 + fi + +} + +get_user_options $@ +ensure_user_options_are_ok +create_pid +create_lock +ensure_src_is_sane +ensure_target_dir_is_there +sync_it +print_report +remove_lock +tidy_up +remove_pid diff --git a/usr/bin/system-info b/usr/bin/system-info index 3df03f28..bc1297c0 100755 --- a/usr/bin/system-info +++ b/usr/bin/system-info @@ -1,193 +1,394 @@ #! /usr/bin/env bash -# Getting system info, especially useful before reporting to Escenic -# select Support. - -# can be: ascii, confluence -output_format=ascii -on_debian_or_derivate=0 -on_gentoo_or_derivate=0 -on_redhat_or_derivate=0 -on_linux=0 - -important_packages_on_debian=" -ant -apache2 -libapr1 -libmysql-java -libtcnative-1 -maven2 -mysql-server -nginx -percona-server-server -slapd -sun-java6-jdk -sun-java6-jre -tomcat6 -tomcat6-user -varnish -" -important_packages_on_gentoo=" -dev-db/percona-server -dev-java/ant-contrib -dev-java/ant-nodeps -dev-java/maven-bin -dev-libs/apr -net-misc/memcached -net-nds/openldap -virtual/jre-1.6.0 -www-servers/apache -www-servers/nginx -www-servers/varnish -" - -if [ `uname -s` = "Linux" ]; then - on_linux=1 -fi - -if [ -x /usr/bin/dpkg -a -e /etc/debian_version ]; then - on_debian_or_derivate=1 -elif [ -x /usr/bin/emerge -a -e /etc/gentoo-release ]; then - on_gentoo_or_derivate=1 -fi - - -function print_ruler() -{ - if [ $output_format = "ascii" ]; then - for i in {0..72}; do - echo -n $1 - done - fi +## Script which creates an overview of system information related to +## the specified ECE/EAE/Search instance, example invocation: +## +## $ system-info -i engine1 | xmllint --format - > /var/www/engine1.html +## +## Add this to /etc/cronttab if you want it to create an HTML report +## every minute: +## +## echo '* * * * * root system-info -f html > /var/www/index.html' >> /etc/crontab + +ece_user="" +ece_scripts_version="straight-from-github" + +## possible values: html, org, confluence, yaml, json (not complete) +format=yaml +generate_output_file_per_module=0 +output_dir=$(pwd) +output_file="" + +current_indent_level=0 +INDENT=" " +verbose=0 +temporaries=1 +log=$HOME/.$(basename $0).log + +function init() { + # first, try to be nice + ece_scripts_dir=$(dirname $0)/../share/escenic/ece-scripts + + # then check the standard location + if [ ! -d $ece_scripts_dir ]; then + ece_scripts_dir=/usr/share/escenic/ece-scripts + fi + + source $ece_scripts_dir/common-ece.sh + source $ece_scripts_dir/common-bashing.sh + source $ece_scripts_dir/common-io.sh } -function print_pre_start() -{ - if [ $output_format = "confluence" ]; then - echo "{code}" +## Runs extra system-info modules +function run_system_info_modules() { + if [ -d $ece_scripts_dir ]; then + # load system-info modules + if [ ! -d $ece_scripts_dir/system-info.d ]; then + return fi + + for el in $(\ls $ece_scripts_dir/system-info.d/*.sh 2>/dev/null); do + if [ $generate_output_file_per_module -eq 1 ]; then + source $el | tee ${output_dir}/$(basename $el).${format} + else + source $el + fi + done + fi } -function print_pre_end() -{ - if [ $output_format = "confluence" ]; then - echo "{code}" - fi +## $1 : indent level, optional, if not set, will use the +## current_indent_level +function get_indent() { + local result="" + + if [[ "${1}x" != "x" ]]; then + local number_of_indents=$1 + else + local number_of_indents=$current_indent_level + fi + + for (( i = 0; i < $number_of_indents; i++ )); do + result="${INDENT}$result" + done + + echo "$result" } -function print_header() +function create_header() { + local title="Overview of $HOSTNAME" + + if [ $format == "html" ]; then +cat <<EOF + +<html xmlns="http://www.w3.org/1999/xhtml"> + <head> + <title>$title @ $(date) + + + +$(cat $ece_scripts_dir/vizrt-logo-svg.html) +EOF + elif [ $format == "json" ]; then + cat </dev/null | grep ^ii | sed 's/ii\ \ //g' - done - elif [ $on_gentoo_or_derivate -eq 1 ]; then - for el in $important_packages_on_gentoo; do - equery --no-color --no-pipe list $el 2>/dev/null | \ - grep -v "Searching for" | \ - cut -d']' -f3- 2>/dev/null - done +function create_footer() { + if [ $format == "html" ]; then + cat < + +EOF + + elif [ $format == "json" ]; then + cat <$el" + else + result="$result $el" + fi + done + else + result="$@" + fi + + echo "$result" } -function list_os_info() -{ - print_header "Kernel version" - uname -a - - print_header "Distribution information" - if [ $on_debian_or_derivate -eq 1 ]; then - echo "Debian or derivate, version "`cat /etc/debian_version` - elif [ -r /etc/gentoo-release ]; then - cat /etc/gentoo-release - fi +function print_un_ordered_list_start() { + if [ $format == "org" ]; then + echo "" + elif [ $format == "html" ]; then + echo "
    " + elif [ $format == "confluence" ]; then + echo "" + elif [ $format == "json" ]; then + echo " [" + fi } -function list_db_information() -{ - print_header "Database details" - if [ -x /usr/sbin/mysqld ]; then - /usr/sbin/mysqld -V - else - mysql -V 2>/dev/null - mysql5 -V 2>/dev/null - fi +function print_un_ordered_list_end() { + if [ $format == "org" ]; then + echo "" + elif [ $format == "html" ]; then + echo "
" + elif [ $format == "confluence" ]; then + echo "" + elif [ $format == "json" ]; then + echo " ]," + fi } -function list_java_information() -{ - print_header "Java version" - java -version +function print_list_item() { + if [ $format == "org" ]; then + echo "- $@" + elif [ $format == "html" ]; then + echo "
  • " \ + "$@" \ + "
    • " + elif [ $format == "confluence" ]; then + echo "* $@" + elif [ $format == "json" ]; then + echo " \"$@\", " + elif [ $format == "yaml" ]; then + echo "$(get_indent) - $@" + fi } -function list_hardware_information() -{ - if [ $on_linux -eq 1 ]; then - print_header "Processors" - echo "Processor type: " `grep "model name" /proc/cpuinfo | head -1 | cut -d':' -f2` - echo "Number of processors/cores:" `grep processor /proc/cpuinfo | wc -l` - - print_header "Memory" - echo "Total memory: " `grep MemTotal /proc/meminfo | cut -d':' -f2` - echo "Free memory: " `grep MemFree /proc/meminfo | cut -d':' -f2` - fi +function json_escape_string() { + echo "$@" | sed 's/\"/\\"/g' +} - print_header "Disk Storage" - print_pre_start - if [ $(df --version | grep GNU | wc -l) -gt 0 ]; then - df -hT - else - df - fi - print_pre_end +function print_pre_text() { + if [ $format == "org" ]; then + cat < +$@ + +EOF + elif [ $format == "confluence" ]; then + cat <$@" + elif [ $format == "confluence" ]; then + echo "h3. $@" + elif [ $format == "json" ]; then + cat <$@" + elif [ $format == "confluence" ]; then + echo "h4. $@" + elif [ $format == "yaml" ]; then + echo "" + echo "$(get_indent 3)${@}:" + current_indent_level=3 + fi +} + +function print_h2_header() { + if [ $format == "org" ]; then + echo "" + echo "** $@" + elif [ $format == "html" ]; then + echo "

    $@

    " + elif [ $format == "confluence" ]; then + echo "h2. $@" + elif [ $format == "json" ]; then + cat <$@" + elif [ $format == "confluence" ]; then + echo "h1. $@" + elif [ $format == "yaml" ]; then + echo "${@}:" + current_indent_level=0 + fi } -for el in $@; do - if [ $el = "-c" -o $el = "--confluence" ]; then - output_format=confluence +function print_p_text() { + if [ $format == "org" ]; then + echo -e "$@" + elif [ $format == "html" ]; then + cat < + $@ +

    +EOF + elif [ $format == "confluence" ]; then + echo -e "$@" + elif [ $format == "json" ]; then + echo " \"message\": \"$text\"," + elif [ $format == "yaml" ]; then + local LINE_WRAP=72 + local line="" + text="" + + for el in $@; do + local tmp_line="${line} ${el}" + # wrap the line when it becomes too long + if [ ${#tmp_line} -gt $LINE_WRAP ]; then + text="${text}\n$(get_indent)${el}" + line="" + else + line="${line} ${el}" + fi + done + + # adding the last line line here. For one liners, text will be + # empty here, so the indent is straght on. For multi line input, + # the text variable already contains a new line at this point. + text="${text}\n$(get_indent)${line}" + echo -e "$text" + fi +} + +function get_user_options() { + # starting to add support for --long-version options + for el in $@; do + if [ $el = "-V" -o $el = "--version" ]; then + echo "Version:" $ece_scripts_version + exit 0 fi -done + done + + while getopts ":d:o:i:f:u:mvst" opt; do + case $opt in + v) + verbose=1 + ;; + s) + temporaries=0 + ;; + d) + output_dir=${OPTARG} + mkdir -p $output_dir + ;; + u) + ece_user=${OPTARG} + ;; + f) + format=${OPTARG} + ;; + o) + output_file=${OPTARG} + ;; + m) + generate_output_file_per_module=1 + ;; + t) + generate_import_job_overview=1 + ;; + \?) + echo "Invalid option: -$OPTARG" >&2 + exit 1 + ;; + :) + echo "Option -$OPTARG requires an argument." >&2 + exit 1 + ;; + esac + done -print_report_header -list_os_info -list_db_information -list_java_information -list_useful_package_info -list_hardware_information +} - - \ No newline at end of file +init +get_user_options $@ +create_header +run_system_info_modules +create_footer diff --git a/usr/bin/vosa b/usr/bin/vosa new file mode 100755 index 00000000..a33ef941 --- /dev/null +++ b/usr/bin/vosa @@ -0,0 +1,1179 @@ +#!/bin/bash + +# /usr/bin/vosa --- command to manage vizrt on-line system-administration +# managed instances of virtual machines. + +# vosa list --- list all available virtual machines +# vosa -i /etc/vizrt/vosa/available.d/vm03 enable --- enable a specific vm +# vosa -i /etc/vizrt/vosa/available.d/vm03 disable --- disable a specific vm +# vosa -i /etc/vizrt/vosa/enabled.d/vm03 disable --- disable a vm +# vosa -i /etc/vizrt/vosa/enabled.d/vm03 install --- creates a new disk image etc. +# vosa -i /etc/vizrt/vosa/enabled.d/vm03 uninstall --- removes the disk image etc. +# vosa -i ... start +# vosa -i ... status --- tells you about the VM, if it's enabled, running, alive, its uptime. + +instance_dir= +instance_name= +usage=0 +ece_scripts_version="straight-from-github" + +### To add an option, add it to the optstring, as defined in getopts, in alphabetical order. +### Also add a function "option-x" where x is the option. $1 will be the option value, if any. +### set any variables needed to default values as globals first. +OPTSTRING=":i:v:hdV" + +function option-h() { + usage=1 +} + +function option-d() { + debug=$((debug + 1)) +} + +function option-V() { + echo "Version:" $ece_scripts_version + exit 0 +} + +function option-i() { + if [[ "${1}" =~ ^[-a-z0-9]*$ ]] ; then + instance_dir=${available_dir}/$1 + else + instance_dir="${1}" + fi + if [ ! -r $instance_dir ] ; then + echo "Instance $1 does not exist. Exiting" + exit 1 + else + instance_dir=$(readlink -f "$instance_dir") + instance_name=$(basename $instance_dir) + fi +} + +function option-v() { + uec_version="${1}" +} + + +function unknown-option() { + echo "Unknown option $@" + usage=1 +} + +### To add a command, simply define a function with a do_ prefix. +available_dir=$(dirname $0)/../../etc/vizrt/vosa/available.d +available_dir=$(readlink -f ${available_dir}) +enabled_dir=$(dirname $0)/../../etc/vizrt/vosa/enabled.d +enabled_dir=$(readlink -f ${enabled_dir}) + + +function requires_instance_dir() { + if [ -z "$instance_dir" -o -z "${instance_name}" ] ; then + echo "Instance is required" + usage=1 + return 1 + fi + + aws=0 + [ -r $instance_dir/amazon.conf ] && aws=1 + return 0 +} +function prohibits_instance_dir() { + if [ ! -z "$instance_dir" -o ! -z "${instance_name}" ] ; then + echo "Instance cannot be specified" + usage=1 + return 1 + fi +} + +## Provides instant gratification. +function do_help() { + cat < /etc/vizrt/vosa/skeleton-kvm/install.conf < /etc/vizrt/vosa/skeleton-kvm/boot.conf < /etc/vizrt/vosa/skeleton-amazon/amazon.conf </ +## unless it already exists. +## +## Required option -v naming the distribution, e.g. oneiric or lucid. +## +## Also changes the symlink of "current" to point to this image. +function do_download() { + if [ -z "$uec_version" ] ; then + echo "-v option is required when downloading." + exit 2 + fi + date=$(date --iso) + local dir + dir=/var/lib/vizrt/vosa/uec-images/$uec_version-$date/ + if [ -d $dir ] ; then + echo "$uec_version-$date already exists, skipping download." + echo "Remove $dir if you want to re-download this image." + else + mkdir -p $dir || exit 2 + wget http://uec-images.ubuntu.com/server/$uec_version/current/$uec_version-server-cloudimg-amd64.tar.gz -O $dir/$uec_version-server-cloudimg-amd64.tar.gz || exit 2 + tar -x --no-same-owner --no-same-permissions -C $dir -v -f $dir/$uec_version-server-cloudimg-amd64.tar.gz | tee $dir/contents > /dev/null || exit 2 + local kernel + kernel=$(cd $dir && echo *-vmlinuz-*) + image=$(cd $dir && echo *.img) + mv $dir/$kernel $dir/vmlinuz + mv $dir/$image $dir/disk.img + fi + if [ -L /var/lib/vizrt/vosa/uec-images/current ] ; then + rm /var/lib/vizrt/vosa/uec-images/current + fi + echo "Marking $uec_version-$date as"' "current"' + ln -s $uec_version-$date /var/lib/vizrt/vosa/uec-images/current +} + +## Provides even more help. +function do_longhelp() { + cat </ + +Each instance needs an 'install.conf' and 'boot.conf' which +describes installation and boot parameters, like ip address, +memory and disk space, and so on. + +An instance may be "enabled" if it is desirable that an instance +definition actually be used on this installation of vosa. + +The idea behind available and enabled instances is that a vosa +cluster will consist of many vosa hosts. For example, 15 different +guests might be running on four different hosts. All 15 definitions +should be present as "available" on all four hosts, but only +the three or four that _should be working_ should be "enabled" for +a host. + +EOF + echo "$0 supports these commands:" + declare -F | grep ^"declare -f do_" | cut -d ' ' -f3 | cut -d'_' -f2 | sort | \ + while read f; do + echo -n ' * '$f: # first line on the same as the command. + awk -f $commands/help.awk fn=do_$f < $0 + echo + done +} + +## List detailed information on how to use all the commands. +function do_commands() { + declare -F | grep ^"declare -f do_" | cut -d ' ' -f3 | cut -d'_' -f2 | sort | \ + while read f; do + echo -n ' * '$f: # first line on the same as the command. + awk -f $commands/help.awk fn=do_$f < $0 | head -n 1 + done +} + +function verify_installation() { + if [ -z "$available_dir" -o ! -r "$available_dir" ] ; then + echo "Not initialized! RTFM! Exiting. ($available_dir)" + exit 2 + fi + + if [ -z "$enabled_dir" -o ! -r "$enabled_dir" ] ; then + echo "Not initialized! RTFM! Exiting. ($enabled_dir)" + exit 2 + fi +} + +## Provides status information about defines VMs +## +## vosa status +## Provides status information on all known VMs. +## +## vosa -i status +## Provides status information on a single VM +## +## Example output: +## vm01 +## vm02 enabled +## vm03 enabled installed +## vm04 enabled installed alive 31374 4-12:45:06 +## vm05 enabled installed dead +## +## The example output shows all possible status. +## +## An instance is defined to be the existsence of a +## directory /etc/vizrt/vosa/available.d// with +## configuration information about the instance. +## +## Status information consists of up to 6 columns +## - instance name +## The first column is the instance name itself. +## An instance may be enabled (see "vosa enable") +## - enabled +## Only enabled instances are "supposed" to be managed by this +## machine. +## An enabled instance *is* supposed to be managed by this +## machine. +## An enabled instance may later be be installed. (see +## "vosa install") +## - installed +## An installed instance has an image directory, created by +## "vosa install" +## An installed instance may be started (see "vosa start"). +## An installed instance may be uninstalled (see "vosa +## uninstall"). +## - alive or dead (actual run state) +## If any of the words alive or dead are present in the fourth +## column then that means that the instance _should_ be running. +## An instance that has been started will probably be alive, but +## if the instance is killed or dies for whatever reason other +## than the "vosa stop" command, it will be labeled as "dead". +## - pid (of the kvm instance) or instance-id (of an amazon VM) +## For kvm instances, the pid is printed. +## For ec2 instances, the instance-id and availability zone is +## printed, separated by a slash. +## - uptime (of any alive instance) +## Only shown for instances that are "alive". +## uptime is shown in dd-hh:mm:ss format. +## AWS instances print the start date (TODO: fix this discrepancy) +function do_status() { + verify_installation + if [ -z "$instance_dir" ] ; then + local a + for a in $( + ls 2>/dev/null -d ${available_dir}/* | grep "/[0-9a-z][-0-9a-z]*$" + ) ; do + really_do_status $a + done + else + really_do_status $instance_dir + fi +} + + +have_read_amazon_config=0 +function read_amazon_config() { + [[ $have_read_amazon_config -eq 1 ]] && return + source $commands/functions + source $commands/amazon_config_parser + # Parse all amazon config items + parse_config_file $1 amazon_config_ + + unset AWS_COMMON_OPTIONS + if [ ! -z "$amazon_config_key" ] ; then + AWS_COMMON_OPTIONS="${AWS_COMMON_OPTIONS} --private-key $amazon_config_key " + AWS_COMMON_OPTIONS="${AWS_COMMON_OPTIONS} --cert $amazon_config_certificate " + AWS_COMMON_OPTIONS="${AWS_COMMON_OPTIONS} --region $amazon_config_region " + export USE_AWS_CLI=false + else + export AWS_ACCESS_KEY_ID=$amazon_config_access_key_id + export AWS_SECRET_ACCESS_KEY="$(cat $amazon_config_secret_access_key_file)" + export AWS_DEFAULT_REGION="$amazon_config_region" + export USE_AWS_CLI=true + AWS_COMMON_OPTIONS="${AWS_COMMON_OPTIONS} --output text " + fi + have_read_amazon_config=1 +} + +function really_do_status () { + local output + local done=0 + local aws=0 + local avdir="${available_dir}/$(basename $1)" + local endir="${enabled_dir}/$(basename $1)" + output=( $(basename $1) ) + if [ ! -d "$avdir" ] ; then + done=1 + fi + + if [ $done -eq 0 ] ; then + if [ ! -L "$endir" ] ; then + output=( ${output[@]} disabled ); + done=1 + fi + fi + + if [ $done -eq 0 ] ; then + output=( ${output[@]} enabled ); + local vmdir=$(readlink -f ${avdir}/../../../../../var/lib/vizrt/vosa/images/$(basename $1)) + if [ ! -d "$vmdir" ] ; then + done=1 + fi + fi + + if [ $done -eq 0 ] ; then + if [ -r $avdir/amazon.conf ] ; then + aws=1 + read_amazon_config $avdir/amazon.conf + fi + + if [ $aws -eq 0 ] ; then + output=( ${output[@]} installed ); + local pidfile=$(echo_rundir_of $(basename $1))/$(basename $1).pid + if [ ! -r "$pidfile" ] ; then + done=1 + fi + else + local bootstatefile=$vmdir/amazon.initialstate + local statefile=$vmdir/amazon.state + if [ ! -r $bootstatefile ] ; then + output=( ${output[@]} unknown ); + # MUST have state to know the instance ID in amazon... + done=1 + fi + + local aws_instance= + if [ $done -eq 0 ] ; then + # Parse the statefile and find its instance ID + if [ -r $bootstatefile ] ; then + aws_instance=$(awk < "$bootstatefile" -F '\t' '/^INSTANCE/ { print $2 }') + fi + if [ -z $aws_instance ] && [ -r $bootstatefile ] ; then + aws_instance=$(cut < "$bootstatefile" -f 6 | grep 'i-') + fi + if [ -z "$aws_instance" ] && [ -r $bootstatefile ] ; then + # Hope that the 6th field continues to provide "i-"... + aws_instance=$(cut < "$bootstatefile" -f 8 | grep 'i-') + fi + + if [ -z "$aws_instance" ] ; then + output=( ${output[@]} invalid ); + done=1 + fi + fi + if [ $done -eq 0 ] ; then + output=( ${output[@]} installed ); + fi + fi + fi + + if [ $done -eq 0 ] ; then + if [ $aws -eq 0 ] ; then + if ! ps > /dev/null $(<$pidfile) ; then + output=( ${output[@]} dead ); + else + output=( ${output[@]} alive ) + output=( ${output[@]} $(<$pidfile) ); + output=( ${output[@]} "$( ps -p "$(<$pidfile)" -o "etime=" )" ); + fi + else + local rc=0 + # Update the aws statefile + if [ -w $statefile ] ; then + if $USE_AWS_CLI ; then + aws $AWS_COMMON_OPTIONS ec2 describe-instances --instance-ids $aws_instance > $statefile + else + ec2-describe-instances $AWS_COMMON_OPTIONS $aws_instance > $statefile + fi + local rc=$? + fi + if [ ! -r $statefile ] ; then + statefile=$bootstatefile + local stale=1 + fi + if $USE_AWS_CLI ; then + aws_state=$(awk < "$statefile" -F '\t' $'/^STATE\t/ { print $3 }') + aws_zone=$(awk < "$statefile" -F '\t' $'/^PLACEMENT\t/ { print $4 }') + aws_start=$(awk < "$statefile" -F '\t' "/$aws_instance/"' { print $4 }') + else + aws_state=$(awk < "$statefile" -F '\t' '/^INSTANCE/ { print $6 }') + aws_zone=$(awk < "$statefile" -F '\t' '/^INSTANCE/ { print $12 }') + aws_start=$(awk < "$statefile" -F '\t' '/^INSTANCE/ { print $11 }') + fi + if [ -z "$aws_state" ] ; then + output=( ${output[@]} unknown $aws_instance ) + else + output=( ${output[@]} ${aws_state} ${aws_zone}/${aws_instance} ) + output=( ${output[@]} ${aws_start} ) + fi + if [ "$stale" == "1" ] ; then + output=( ${output[@]} STALE) + fi + fi + fi + echo ${output[@]} +} + +function echo_rundir_of() { + local avdir="${available_dir}/$(basename $1)" + echo "$(readlink -f ${avdir}/../../../../../var/run/vizrt/vosa/)" +} + +## Tells vosa that it is allowed to install and run an instance. +## +## vosa -i enable +## Enables on this host. +## +## vosa will create a symlink in /etc/vizrt/vosa/enabled.d/ +## pointing to "../available.d/". This is all that is +## required to enable an instance. +## +## An enabled instance can be installed. +function do_enable() { + requires_instance_dir && { + # check if the instance is available + # check if the instance isn't enabled already + # check if the instance isn't forcibly disabled + # make a symbolic link + if [ ! -d ${available_dir}/$instance_name ] ; then + echo "$instance_name is not an available instance" + exit 1 + fi + if [ -L ${enabled_dir}/$instance_name ] ; then + echo "$instance_name is already enabled" + exit 1 + fi + if [ -r ${enabled_dir}/$instance_name ] ; then + echo "$instance_name has been forcibly disabled by the presence of the file ${enabled_dir}/$instance_name" + exit 1 + fi + ln -v -s "../available.d/$instance_name" "$enabled_dir" || exit 1 + } +} + + +## Tells vosa that it should ignore an instance definition +## +## vosa -i disable +## Disables on this host. +## +## vosa will remove the symlink in /etc/vizrt/vosa/enabled.d/ +## pointing to "../available.d/". This is all that is +## required to disable an instance. +## +## An disabled instance can not be installed, but may be enabled +## later. +function do_disable() { + requires_instance_dir && { + if [ ! -d ${available_dir}/$instance_name ] ; then + echo "$instance_name is not an available instance" + exit 1 + fi + if [ ! -L ${enabled_dir}/$instance_name ] ; then + echo "$instance_name is not enabled" + exit 1 + fi + rm "${enabled_dir}/$instance_name" || exit 1 + echo "$instance_name has been disabled" + } +} + + + +## Removes the installed image of an instance +## +## This deletes the contents of the directory +## /var/lib/vizrt/vosa/images// if it exists. +## An amazon backed instance is terminated and cannot be recovered. +## +## Removing the installed image of an instance makes it impossible to +## start the instance unless it is (re)installed later. Uninstalling +## an instance is destructive, but if the directory +## /var/backup/vizrt/vosa/ +## exists then ONE generation of backups will be kept in that directory, +## and it will be possible to restore an old backup by manually moving +## the backup directory back into /var/lib/vizrt/vosa/images/ +## +## Amazon EC2 instances will be completely lost, as the instances are +## terminated, meaning they go away and cannot be revived. +## If an instance has "termination protection" turned on, then uninstalling +## this way will not work. Enable termination protection to make it +## impossible to accidentally uninstall a valuable image. +## +## It is not possible to uninstall an instance that is running, or +## that _should_ be running (i.e. if a .pid file exists). +## +## Please be careful with uninstall. +function do_destroy() { + requires_instance_dir && { + # check if the instance is available + # check if the instance isn't enabled already + # check if the instance isn't forcibly disabled + # make a symbolic link + if [ ! -d ${available_dir}/$instance_name ] ; then + echo "$instance_name is not an available instance" + exit 1 + fi + if [ ! -L ${enabled_dir}/$instance_name ] ; then + echo "$instance_name is not enabled" + exit 1 + fi + + [ -r "$instance_dir/amazon.conf" ] && read_amazon_config $instance_dir/amazon.conf + local status=( $(really_do_status $instance_dir) ) + if [ "${status[3]}" == "alive" -o "${status[3]}" == "running" ] ; then + echo "$instance_name is alive. You might want to stop instances before destroying them." + fi + if [ "${status[3]}" == "dead" ] ; then + echo "$instance_name is dead, but supposed to be running. Stop it first. Exiting!" + exit 1 + fi + local imgdir=$(readlink -f ${instance_dir}/../../../../../var/lib/vizrt/vosa/images/$instance_name/) + local backupdir="/var/backup/vizrt/vosa/$instance_name" + local backupparent="/var/backup/vizrt/vosa" + if [ $aws -eq 1 ] ; then + ${commands}/ec2-cmd.sh \ + "$instance_dir" \ + "$(dirname $0)/../../var/lib/vizrt/vosa/images/$instance_name" \ + "terminate-instances" $($USE_AWS_CLI && echo "--instance-ids") "INSTANCE" || exit $? + # If the terminate-instances command fails, don't remove the instance directory + # But what if it's terminated twice? + fi + if [ ! -z "$imgdir" -a -d "$imgdir" ] ; then + if [ -w "$backupparent" ] ; then + # todo: rotate a few times? + rm -rf "$backupdir" + mv "$imgdir" "$backupparent" + else + rm -rf "$imgdir" + fi + fi + } +} + + +commands="$(readlink -f "$(dirname $0)")/../share/vizrt/vosa/commands" + + +## Installs an image of an instance +## +## This creates a disk image and supporting files in the directory +## /var/lib/vizrt/vosa/images//, starts a virtual machine +## and configures it as described in the instance configuration +## directory /etc/vizrt/vosa/available.d//. +## +## Installing an instance usually ends up in a functional virtual +## machine. +## +## It is not possible to install an instance that has already been +## installed. +function do_create() { + requires_instance_dir && { + local status=( $(really_do_status $instance_dir) ) + if [ "${status[2]}" == "installed" ] ; then + echo "$instance_name is already installed. Uninstall before installing." + exit 1 + fi + if [ "${status[1]}" != "enabled" ] ; then + echo "$instance_name is not enabled, enable it on this host first" + exit 1 + fi + if [ ! -d "$(dirname $0)/../../var/lib/vizrt/vosa/images" ] ; then + mkdir -p "$(dirname $0)/../../var/lib/vizrt/vosa/images" + fi + if [ $aws -eq 1 ] ; then + mkdir "$(dirname $0)/../../var/lib/vizrt/vosa/images/$instance_name" || exit 2 + read_amazon_config $instance_dir/amazon.conf + # TODO: sanitize "required" variables + if $USE_AWS_CLI ; then + json="[{\"device_name\":\"/dev/sda1\", \"ebs\":{\"volume_size\":$amazon_config_initial_disk_size} } ]" + opts=( "--image-id" ${amazon_config_image} + "--min-count" "1" + "--max-count" "1" + ${amazon_config_availability_zone/#/--placement availability_zone=} \ + ${amazon_config_type/#/--instance-type } \ + ${amazon_config_ssh_keypair/#/--key-name } \ + ${amazon_config_group/#/--security-groups } \ + --block-device-mapping "${json}" \ + ${amazon_config_subnet/#/--subnet-id } + ) + if [ ! -z ${amazon_config_user_data} ] ; then + opts=( "${opts[@]}" + --user-data "$(cat $amazon_config_user_data)" + ) + fi + + else + opts=( + ${amazon_config_image} \ + ${amazon_config_availability_zone/#/--availability-zone } \ + ${amazon_config_type/#/--instance-type } \ + ${amazon_config_ssh_keypair/#/--key } \ + ${amazon_config_group/#/--group } \ + ${amazon_config_user_data/#/--user-data-file } \ + ${amazon_config_subnet/#/--subnet } + ) + fi + + + ${commands}/ec2-cmd.sh \ + "$instance_dir" \ + "$(dirname $0)/../../var/lib/vizrt/vosa/images/$instance_name" \ + run-instances \ + "${opts[@]}" + + if [ ! -z "${amazon_config_ip_address}" ] ; then + echo "Associating IP $amazon_config_ip_address..." + for a in 1 2 3 4 5 6 7 8 9 10; do + sleep 4; + ${commands}/ec2-cmd.sh \ + "$instance_dir" \ + "$(dirname $0)/../../var/lib/vizrt/vosa/images/$instance_name" \ + associate-address \ + $($USE_AWS_CLI && echo "--public-ip") \ + ${amazon_config_ip_address} \ + $($USE_AWS_CLI && echo "--instance-id") \ + $($USE_AWS_CLI || echo "--instance") \ + INSTANCE && break + done + fi + + ${commands}/ec2-determine-ip.sh \ + "$instance_dir" \ + "$(dirname $0)/../../var/lib/vizrt/vosa/images/$instance_name" + + else + ${commands}/install.sh \ + "$instance_dir" \ + "$(dirname $0)/../../var/lib/vizrt/vosa/images/$instance_name" \ + 2>&1 | + tee /var/log/vosa-install-$instance_name.log + local rc=${PIPESTATUS[0]} + if [ "$rc" != "0" ] ; then + exit $rc + fi + fi + } +} + +## Installs software to a newly provisioned instance +## +## All postinst hooks defined in the instance's postinst hook +## will be run. It should only be necessary to run this command +## once, as it should run to completion giving you a completely +## installed machine. +## +## It is possible to postinst an instance that has already been +## postinst'ed. It is only possible to postinst an instance +## that is running. +function do_install() { + requires_instance_dir && { + local status=( $(really_do_status $instance_dir) ) + if [ "${status[2]}" != "installed" ] ; then + echo "$instance_name is not installed. Install before running any postinst." + exit 1 + fi + if [ "${status[3]}" != "alive" -a "${status[3]}" != "running" ] ; then + echo "$instance_name is not alive/running, start it first." + exit 1 + fi + ${commands}/postinst.sh \ + "$instance_dir" \ + "$(dirname $0)/../../var/lib/vizrt/vosa/images/$instance_name" \ + 2>&1 | + tee -a /var/log/vosa-postinst-$instance_name.log + local rc=${PIPESTATUS[0]} + if [ "$rc" != "0" ] ; then + exit $rc + fi + } +} + +## Starts a stopped instance. +## +## Starts an installed instance that has been stopped or has died +## (or has been shut down from within the guest). A pidfile will be +## created, with the pid of the kvm process, which also serves as a +## flag that the "desired state" of the VM is to be started. +## +## It is not possible to start an instance that has not been installed, +## or that is already running and alive. It _is_ possible to (re)start +## a dead instance. +function do_start() { + # TODO: let's make this command wait for SSH access before returning? + requires_instance_dir && { + local status=( $(really_do_status $instance_dir) ) + if [ "${status[3]}" == "alive" -o "${status[3]}" == "running" ] ; then + echo "$instance_name is ${status[3]}. No need to start it." + exit 1 + fi + if [ "${status[2]}" != "installed" ] ; then + echo "$instance_name is not installed, it's ${status[2]}. I can't start it." + exit 1 + fi + if [ $aws -eq 1 ] ; then + read_amazon_config $instance_dir/amazon.conf + ${commands}/ec2-cmd.sh \ + "$instance_dir" \ + "$(dirname $0)/../../var/lib/vizrt/vosa/images/$instance_name" \ + "start-instances" \ + $($USE_AWS_CLI && echo "--instance-ids") \ + INSTANCE + + if [ ! -z "${amazon_config_ip_address}" ] ; then + echo "Associating IP $amazon_config_ip_address..." + for a in 1 2 3 4 5 6 7 8 9 10; do + sleep 4; + ${commands}/ec2-cmd.sh \ + "$instance_dir" \ + "$(dirname $0)/../../var/lib/vizrt/vosa/images/$instance_name" \ + associate-address \ + $($USE_AWS_CLI && echo "--public-ip") \ + ${amazon_config_ip_address} \ + $($USE_AWS_CLI && echo "--instance-id") \ + $($USE_AWS_CLI || echo "--instance") \ + INSTANCE && break + done + fi + + ${commands}/ec2-determine-ip.sh \ + "$instance_dir" \ + "$(dirname $0)/../../var/lib/vizrt/vosa/images/$instance_name" + else + ${commands}/boot.sh "$instance_dir" "$(dirname $0)/../../var/lib/vizrt/vosa/images/$instance_name" || exit $? + fi + } +} + + +## Stops a running instance. +## +## Stops an instance that has previously been been started. If the instance +## has died, the pidfile will be removed to indicate that the desired state +## is to be stopped. +## +## It is not possible to stop an instance that has not been installed. Stopping +## an already stopped instance has no effect. +function do_stop() { + requires_instance_dir && { + local status=( $(really_do_status $instance_dir) ) + if [ "${status[2]}" != "installed" ] ; then + echo "$instance_name is not installed, it's ${status[2]}. I can't stop it." + exit 1 + fi + if [ "${status[3]}" == "dead" ] ; then + local pidfile=$(echo_rundir_of ${status[0]})/${status[0]}.pid + rm -f $pidfile + fi + if [ "${status[3]}" == "alive" -o "${status[3]}" == "running" ] ; then + shutdown "${status[0]}" + # should now be "dead" if we asked it. + fi + #if [ $aws -eq 0 ] ; then + # regardless of this, we _want_ it to die. remove pidfile. + # TODO: continue trying to kill it more forcibly, and + # remove the pidfile when it's done. + #local pidfile=$(echo_rundir_of ${status[0]})/${status[0]}.pid + #rm -f $pidfile + #fi + } +} + +# Performs a graceful shutdown of the guest +function shutdown_kvm() { + local pidfile="$(echo_rundir_of $1)/$1.pid" + local pid=$(<$pidfile) + if [ -z "$pid" ] ; then + rm $pidfile + return 0 + fi + ps > /dev/null $pid && { + echo "Sending ACPI shutdown signal to guest ($pid) ($(echo_rundir_of $1)/$1.monitor)" + echo 'system_powerdown' | nc -U $(echo_rundir_of $1)/$1.monitor + echo + echo -n "Waiting up to 60 for it to shut down gracefully." + } + local a; + for a in $(seq 1 60) ; do + echo -n '.' + sleep 1 + ps > /dev/null $pid || { + echo; + rm $pidfile + return 0; + } + done + ps > /dev/null $pid && { + echo; + echo "Sending quit signal to kvm ($(echo_rundir_of $1)/$1.monitor)" + echo 'quit' | nc -U $(echo_rundir_of $1)/$1.monitor + echo -n "Waiting 30 seconds for it to stop" + } + local a; + for a in $(seq 1 30) ; do + echo -n '.' + sleep 1 + ps > /dev/null $pid || { + echo; + rm $pidfile + return 0; + } + done + ps > /dev/null $pid && { + echo "guest did not shut down. Killing (like pulling power cord)." + kill "${status[4]}" + echo -n "Waiting 10 seconds for it to die" + } + for a in $(seq 1 10) ; do + sleep 1 + ps > /dev/null $pid || { + echo; + rm $pidfile + return 0; + } + done + ps > /dev/null $pid && { + echo "guest did not respond to kill. Killing with -9. Leaving pidfile alone." + kill -9 "${status[4]}" + } +} + +function shutdown_aws() { + read_amazon_config $instance_dir/amazon.conf + ${commands}/ec2-cmd.sh "$instance_dir" "$(dirname $0)/../../var/lib/vizrt/vosa/images/$instance_name" "stop-instances" \ + $($USE_AWS_CLI && echo "--instance-ids") INSTANCE +} + +# Shuts +function shutdown() { + if [ $aws -eq 1 ] ; then + shutdown_aws "${@}" + else + shutdown_kvm "${@}" + fi +} + + +function unknown-argument() { + echo "Unknown argument $@" + usage=1 +} + +LASTOPTIND=0 + +function parseopts() { +local OPTARG +local OPTIND +local opt +local currentconfigdirectory="${@:1:1}" +shift; +while getopts "${OPTSTRING}" opt; do + case $opt in + \?) + unknown-option "-$OPTARG" + ;; + *) + option-$opt "${OPTARG}" + ;; + esac +done + +LASTOPTIND=$OPTIND + +} + +function verify_usage() { + if [ $usage -eq 1 ] ; then + echo "Usage: $0 [-i ] " + echo "$0 help for more help" + exit 1; + fi +} + +parseopts "$PWD" "${@}" + +# get rid of all parsed parameters from command line, leaving real arguments +shift $((LASTOPTIND-1)) + +if [ $usage -eq 0 -a "${#@}" -lt 1 ] ; then + echo "A command must be specified." + usage=1 +fi + +verify_usage + +for cmd in "${@}" ; do + # Check if the command + fn="do_${cmd}" + declare > /dev/null -f "$fn" || unknown-argument "$1" + declare > /dev/null -f "$fn" && "$fn" + verify_usage +done + + + diff --git a/usr/bin/vosa-cat b/usr/bin/vosa-cat new file mode 100644 index 00000000..2d5b1a1a --- /dev/null +++ b/usr/bin/vosa-cat @@ -0,0 +1,105 @@ +#! /usr/bin/env bash + +vosa_dir=/var/lib/vizrt/vosa/files + +remote_diff=0 + +function get_user_options() { + while getopts ":i:f:s:r" opt; do + case $opt in + r) + remote_diff=1 + ;; + i) + instance=${OPTARG} + ;; + f) + vosa_input_file=${OPTARG} + ;; + s) + scm_dir=${OPTARG} + ;; + \?) + echo "Invalid option: -$OPTARG" >&2 + exit 1 + ;; + :) + echo "Option -$OPTARG requires an argument." >&2 + exit 1 + ;; + *) + echo command is $OPTARG + exit 0 + esac + done +} + +get_user_options $@ + +if [ -z "${instance}" ]; then + echo "You must specify instance with $(basename $0) -i " + exit 1 +fi +if [ -z "${vosa_input_file}" ]; then + echo "You must specify vosa input file with $(basename $0) -f " + exit 1 +fi + +function get_vosa_file() { + local file=${vosa_dir}/${instance}/$vosa_input_file + if [ -e ${file} ]; then + echo $file + else + file=${vosa_dir}/common/$vosa_input_file + if [ -e ${file} ]; then + echo $file + fi + fi +} + +function cat_vosa_file() { + local file=$(get_vosa_file) + if [ -e ${file} ]; then + echo "Contents of $file" + echo "====" + cat $file + echo "====" + else + echo "Couldn't find $vosa_input_file in any of the vosa file layers" + exit 1 + fi +} + +function get_scm_file() { + local file=$scm_dir/$vosa_input_file + if [ -e $file ]; then + echo $file + elif [ -e $scm_dir/$instance/$vosa_input_file ]; then + echo $scm_dir/$instance/$vosa_input_file + elif [ -e $scm_dir/common/$vosa_input_file ]; then + echo $scm_dir/common/$vosa_input_file + else + echo $vosa_input_file "couldn't be found under $scm_dir" + exit 1 + fi +} + +function diff_vosa_file() { + local vosa_file=$(get_vosa_file) + diff $vosa_file $(get_scm_file) +} + +function remote_vosa_diff() { + diff -w $(get_vosa_file) <(ssh chnepmledt001 cat /${vosa_input_file}) +} + +if [ ${remote_diff} -eq 1 ]; then + remote_vosa_diff +else + if [ -z "${scm_dir}" ]; then + cat_vosa_file + else + diff_vosa_file + fi +fi + diff --git a/usr/bin/vosa-push b/usr/bin/vosa-push new file mode 100755 index 00000000..399922ad --- /dev/null +++ b/usr/bin/vosa-push @@ -0,0 +1,45 @@ +#! /usr/bin/env bash + +# wrapper around the vosa::push module. Can be removed once vosa -i +# push has been implemented. + +source $(dirname $0)/../share/vizrt/vosa/commands/push.sh +outgoing=0 +instance="" + +function get_user_options() { + while getopts ":i:o" opt; do + case $opt in + i) + instance=${OPTARG} + ;; + o) + outgoing=1 + ;; + \?) + echo "Invalid option: -$OPTARG" >&2 + exit 1 + ;; + :) + echo "Option -$OPTARG requires an argument." >&2 + exit 1 + ;; + *) + echo command is $OPTARG + exit 0 + esac + done +} + +get_user_options $@ + +if [ -z "${instance}" ]; then + echo "You must specify instance with $(basename $0) -i " + exit 1 +fi + +pre_push_changes +set_etc_hosts +push_changes +post_push_changes + diff --git a/usr/lib/check_mk_agent/local/ece b/usr/lib/check_mk_agent/local/ece new file mode 100755 index 00000000..127dacf6 --- /dev/null +++ b/usr/lib/check_mk_agent/local/ece @@ -0,0 +1,58 @@ +#!/bin/bash + +# check_mk plug-in to verify that the Escenic Content Engine instances are +# actually running. If there's something wrong with the pidfile (missing, +# empty) or if the pid is not up (e.g. dead) then it goes critical. +# + +engine_instance_list="" +search_instance_list="" + +if [ ! -r /etc/default/ece ] ; then + exit 0 +fi + +function determine_health() { + local type=$1 + shift + for i in $* ; do + basename="ece_$i" + pidfile=/var/run/escenic/$i.pid + + if [ ! -r $pidfile ] ; then + echo "2 ${basename}_up - The $pidfile is missing" + continue; + fi + + local pid=$(<$pidfile) + if [ -z "$pid" ] ; then + echo "2 ${basename}_up - The $pidfile is empty" + continue; + fi + + if [ ! -d /proc/$pid ] ; then + echo "2 ${basename}_up - The process in $pidfile is not running" + continue; + fi + + nr_open_files=$(ls /proc/$pid/fd | wc -l) + limit_open_files=$(cat /proc/$pid/limits | awk '/^Max open files /{print $4}') + if [ $nr_open_files -le 2456 ]; then + echo "0 ${basename}_filehandles - OK ${basename} is using $nr_open_files of $limit_open_files file handles" + else + if [ $nr_open_files -le 3456 ]; then + echo "1 ${basename}_filehandles - ${basename} is using $nr_open_files of $limit_open_files file handles (warning at 50%)" + else + echo "2 ${basename}_filehandles - ${basename} is using $nr_open_files of $limit_open_files file handles (critical 75%)" + fi + fi + + echo "0 ${basename}_up - OK process $pid" + + done +} + +. /etc/default/ece +determine_health engine $engine_instance_list +determine_health search $search_instance_list + diff --git a/usr/lib/check_mk_agent/local/ece-install-trail b/usr/lib/check_mk_agent/local/ece-install-trail new file mode 100755 index 00000000..16fd2aa8 --- /dev/null +++ b/usr/lib/check_mk_agent/local/ece-install-trail @@ -0,0 +1,72 @@ +#!/bin/bash + +# check_mk checks for the trail files left by ece-install. + +# Trail files contain interesting things which might be useful to monitor, +# including virtual hosts, port numbers and so on that "have been installed" +# on this host (and therefore should be useful to monitor. +# +# trail_virtual_host_something=www.mypublication.com:8080 +# trail_virtual_host_somethingelse=www.somethingelse.com:8081 +# trail_presentation_port=1234 +# trail_editorial_port=1235 + +critical=900 +warning=700 + +function check_trail() { + for el in /var/lib/escenic/ece-install/*.trail /var/lib/ece-install/*.trail ; do + [ -r "$el" ] || continue; + vars=$(diff <(set -a; unset _; env) <(set -a; unset _;source $el; env) | grep '^>' | sed -e s/^..// -e s/=.*//) + source $el + for var in $vars ; do + if [[ $var =~ ^.*_port$ ]] ; then + local name=${var#trail_} + name=${name%_port} + check_port_listening $name ${!var} + check_port_status $name ${!var} + fi + unset $var + done + done +} + +function check_port_listening() { # foo 4040 + local name=$1 + local port=$2 + local listening=$(sudo -n lsof -s TCP:LISTEN -i 4:$port -i 6:$port -n | tail -n +2 | wc -l) + if [ $listening -eq 0 ] ; then + echo "2 port_${name}_${port}_listen - CRITICAL: not listening on $name port $port" + return; + fi + + echo "0 port_${name}_${port}_listen - OK: listening on $name port $port." +} + +function check_port_status() { # foo 4040 + local name=$1 + local port=$2 + local established=$(sudo -n lsof -s TCP:ESTABLISHED -i 4:$port -i 6:$port -n | tail -n +2 | wc -l) + if [ $established -gt $critical ] ; then + echo "2 port_${name}_${port}_established - CRITICAL: $established established connections to on $name port $port" + return; + fi + + if [ $established -gt $warning ] ; then + echo "1 port_${name}_${port}_established - WARNING: $established established connections to on $name port $port" + return; + fi + + echo "0 port_${name}_${port}_established - OK: $established established connections to on $name port $port" +} + +# TODO: make thresholds different based on the type of port (e.g. editorial +# port vs presentation port vs caching port). + +# TODO: monitor the number of SYN_RECV and verify that it's a low number, +# an early warning of slowness in the system. + +# TODO: monitor virtual hosts are accessible using the host names and port +# numbers provided in the .trail files + +check_trail diff --git a/usr/lib/check_mk_agent/local/ftp-feed-sources b/usr/lib/check_mk_agent/local/ftp-feed-sources new file mode 100755 index 00000000..24d9428c --- /dev/null +++ b/usr/lib/check_mk_agent/local/ftp-feed-sources @@ -0,0 +1,51 @@ +#! /usr/bin/env bash + +## check_mk plugin which checks all FTP servers which are set up to +## provie a feed of external data to be imported into ECE. +## +## The script looks in the trail fails left by the ece-install script +## to determine which FTP feeds to check the health of. + +function read_trails() { + for el in $(ls /var/lib/ece-install/*.trail 2>/dev/null); do + source $el + done +} + +function exit_if_no_trails() { + if [ -z "${trail_ftp_feed_list}" ]; then + exit 0 + fi +} + +function check_ftp_feed_sources() { + for el in $trail_ftp_feed_list; do + local file=/etc/cron.hourly/${el}-import + local user=$(grep ^ftp_user $file | cut -d'=' -f2 | sed 's#"##g') + local password=$(grep ^ftp_password $file | cut -d'=' -f2 | sed 's#"##g') + local url=$(grep ^ftp_url $file | cut -d'=' -f2 | sed 's#"##g') + + local headers=$( + wget -O /dev/null \ + --ftp-user $user \ + --ftp-password $password \ + $url 2>&1 >/dev/null + ) + local login_ok=$(echo "$headers" | grep 'Logged in' | wc -l) + local dir_ok=$(echo "$headers" | grep '==> LIST ... done.' | wc -l) + + local key="import_ftp_source_${el}" + + if [ ${login_ok} -eq 1 -a ${dir_ok} -eq 1 ]; then + echo 0 ${key} "-" "The $el FTP import source is OK" + elif [ ${login_ok} -eq 1 -a ${dir_ok} -eq 0 ]; then + echo 2 ${key} "-" "Could not access the directory on ${url}" + elif [ ${login_ok} -eq 0 ]; then + echo 2 ${key} "-" "Could not log into ${url}" + fi + done +} + +read_trails +exit_if_no_trails +check_ftp_feed_sources diff --git a/usr/lib/check_mk_agent/local/varnish b/usr/lib/check_mk_agent/local/varnish new file mode 100755 index 00000000..298bec2e --- /dev/null +++ b/usr/lib/check_mk_agent/local/varnish @@ -0,0 +1,84 @@ +#!/bin/bash + +# Monitors the varnish access log to see if the number of 404s or other +# errors exceed various thresholds. + +# Checks are only made if varnish has START=yes in its /etc/default/varnish. + +# This check_mk plug-in results in checks called +# varnish_health-404 - Monitors the number of 404 responses from the site +# varnish_health-other - Monitors the number of non-404 error responses +# from the site, e.g. 503 etc. +# +# The hardcoded thresholds are: +# >5% 404 -> warning +# >30% 404 -> critical +# >1% other errors -> warning +# >10% other errors -> critical + + +warning404=200 # 404 requests per 1000 (12%) +critical404=300 # 404 requests per 1000 (30%) +warningerr=20 # failed requests per 1000 (2%) +criticalerr=100 # requests per 1000 (10%) +name="varnish_health" + +function determine_health() { + results=( $(varnishncsa -d -F %s | awk 'BEGIN { err404=0; err = 0; ok=0; } { if ($1 == 404) err404 = err404 +1 ; else if ($1 >= 400) err = err + 1; else ok = ok + 1 } END { factor404=int(err404*1000/(ok+err+err404+1)); factorerr=int(err*1000/(ok+err+err404+1)); print factor404 "\t" factorerr "\t" ok "\t" err404 "\t" err } ') ) + if [ ${PIPESTATUS[0]} != 0 ] ; then + echo "3 ${name}-404 - varnishncsa executed with ${PIPESTATUS[0]}" + echo "3 ${name}-other - varnishncsa executed with ${PIPESTATUS[0]}" + exit + fi + + # type factor ok failed warn crit + check "404" ${results[0]} $((${results[2]} + ${results[4]})) ${results[3]} $warning404 $critical404 + check "other" ${results[1]} $((${results[2]} + ${results[3]})) ${results[4]} $warningerr $criticalerr +} + + +#usage "percentage 55 1000" +# echos 5.5% +function percentage() { + local result=$(( $1 * 1000 / $2 )) + local intpart=$(( $result / 10)) + local decpart=".$(( $result % 10))" + if [ $decpart == ".0" ] ; then + decpart= + fi + echo $intpart$decpart +} + +function check() { + message=OK + state=0 + if [ $2 -gt $6 ] ; then + state=2 + message=CRITICAL + elif [ $2 -gt $5 ] ; then + state=1 + message=WARNING + fi + if [ $(( $3 + $4 )) -lt 3000 ] ; then + if [ $state -gt 0 ] ; then + state=0 + message="(Only $(( $3 + $4)) requests sampled; forcing OK) $message" + fi + fi + message="$message: $(percentage $4 "$(($3+$4))")% of requests ($4/$(($3+$4))) responded with HTTP $1" + echo "$state $name-$1 factor=$2|ok=$3|err=$4 $message" +} + +if [ -r /etc/default/varnish ] ; then + ( + . /etc/default/varnish + export START + ) + if [ $START=yes ] ; then + varnishncsa=$(which varnishncsa) + if [ ! -z $varnishncsa ] ; then + determine_health + fi + fi +fi + diff --git a/usr/lib/check_mk_agent/plugins/periodic-check b/usr/lib/check_mk_agent/plugins/periodic-check new file mode 100755 index 00000000..64cb93b8 --- /dev/null +++ b/usr/lib/check_mk_agent/plugins/periodic-check @@ -0,0 +1,13 @@ +#!/bin/bash + +dir="/var/cache/periodic-check" + +#echo "<<>>" + +for file in `ls $dir`; do +# echo "hello: $dir/$file" + if [ -f "$dir/$file" ]; then + echo "<<<$file>>>" + cat "$dir/$file" + fi +done diff --git a/usr/lib/periodic-check/conf/indexer_running.conf b/usr/lib/periodic-check/conf/indexer_running.conf new file mode 100644 index 00000000..f13eccf7 --- /dev/null +++ b/usr/lib/periodic-check/conf/indexer_running.conf @@ -0,0 +1,5 @@ +connect_timeout=3 +max_time=10 +max_warning_diff=300 #5 minutes +max_critical_diff=600 #10 minutes + diff --git a/usr/lib/periodic-check/conf/site_length.conf b/usr/lib/periodic-check/conf/site_length.conf new file mode 100644 index 00000000..604c9eef --- /dev/null +++ b/usr/lib/periodic-check/conf/site_length.conf @@ -0,0 +1,6 @@ +sites="www.escenic.com +www.escenic.com/incoming" +tomcat_port="8040" +host=localhost +connect_timeout=5 +max_time=15 diff --git a/usr/lib/periodic-check/main b/usr/lib/periodic-check/main new file mode 100755 index 00000000..d35a22da --- /dev/null +++ b/usr/lib/periodic-check/main @@ -0,0 +1,26 @@ +#!/bin/bash +. $(dirname $0)/main.conf + +function init { + if [ -f "$state_file" ]; then + old_pid=$(cat $state_file) + if ps &> /dev/null $old_pid ; then + exit + fi + fi + echo "$$" > "$state_file" +} + +init + +for script in `ls $nagios_lib_dir/script/`; do + if [ -f "$nagios_lib_dir/script/$script" -a -x "$nagios_lib_dir/script/$script" ]; then + nagios_data_file="$nagios_cache_dir/$(basename $script)" + tmp_data_file="$nagios_cache_dir/.$(basename $script).tmp" + $nagios_lib_dir/script/$script > $tmp_data_file + sleep 1 + mv $tmp_data_file $nagios_data_file + fi +done + +rm "$state_file" diff --git a/usr/lib/periodic-check/main.conf b/usr/lib/periodic-check/main.conf new file mode 100644 index 00000000..a8dcb9b3 --- /dev/null +++ b/usr/lib/periodic-check/main.conf @@ -0,0 +1,6 @@ +nagios_cache_dir="/var/cache/periodic-check" +nagios_lib_dir="/usr/lib/periodic-check" +state_file="$nagios_lib_dir/periodic-check.state" +nagios_data_file="$nagios_cache_dir/$(basename $0)" +connect_timeout=5 +max_time=20 diff --git a/usr/lib/periodic-check/script/indexer_running b/usr/lib/periodic-check/script/indexer_running new file mode 100755 index 00000000..f59831c1 --- /dev/null +++ b/usr/lib/periodic-check/script/indexer_running @@ -0,0 +1,91 @@ +#!/bin/bash + +# assumptions: indexer is running if head-tail file has max(documentID) or head-tail file is modified in 5 minutes. +# if indexer is running, script will check the time difference between the last modification time of the head-tail file and +# last modification date mentioned in SearchIndex table for the documentID mentioned as a head in head-tail file + +#output +# +# + +this_script=$(basename $0) + +function init { + . "$(dirname $0)/../main.conf" + if [ -f "$(dirname $0)/../conf/$this_script.conf" ]; then + . "$(dirname $0)/../conf/$this_script.conf" + fi +} +head_56pattern=".*head=[0-9-][0-9]*.*" +head_57pattern=".*head=after/[0-9-]*.*" +head_tail_file="/var/lib/escenic/engine/head-tail.index" +max_warning_diff=180 #3 minutes +max_critical_diff=300 #5 minutes +init + +#filehead=$(cat $head_tail_file | grep "head="| sed 's/.*head=after\/\([0-9][0-9]*\).*/\1/') + +filehead=$(cat $head_tail_file | grep "head=") +if [[ $filehead =~ $head_56pattern ]]; then + filehead=$(cat $head_tail_file | grep "head="| sed 's/.*head=\([0-9][0-9]*\).*/\1/') +elif [[ $filehead =~ $head_57pattern ]]; then + filehead=$(cat $head_tail_file | grep "head="| sed 's/.*head=after\/\([0-9][0-9]*\).*/\1/') +fi + + +file_epoch=$(stat --format "%Y" "$head_tail_file") + +dbhead=$(curl --connect-timeout $connect_timeout --max-time $max_time -isX PUT "http://localhost:8080/escenic-admin/pages/database/query.jsp?query=select%20max(documentID)%20from%20SearchIndex" | grep -A 2 ">max(documentID)"| tail -n 1 | sed 's/.*>\([0-9][-0-9]*\).*/\1/') +curr_epoch=$(date +%s) + +function invalid_diff { + #valid difference must be more than 0 + re="^[-0-9][0-9]*$" + if [[ $1 =~ $re ]] && [[ "$1" -ge 0 ]] && [[ $1 -le 10 ]]; then + echo "0" + return 0 + else + echo "1" + return 1 + fi +} + +indexer_running=0 +if [ "$filehead" == "$dbhead" ]; then + indexer_running=1 +else + diff=$(($curr_epoch - $file_epoch)) + diff_min=$(($diff / 60)) + if [ "$diff" -ge "$max_critical_diff" ]; then + indexer_running=-2 + elif [ "$diff" -ge "$max_warning_diff" ]; then + indexer_critical=-1 + else + indexer_running=1 + fi +fi + +if [ "$indexer_running" == 1 ]; then + + #dbdate=$(echo "select entryUpdated from SearchIndex si where documentID=(select max(documentID) from SearchIndex);"| sudo mysql ece5db | tail -n 2| tail -n 1) + + dbdate=$(curl --connect-timeout $connect_timeout --max-time $max_time -isX PUT "http://localhost:8080/escenic-admin/pages/database/query.jsp?query=select%20entryUpdated%20from%20SearchIndex%20si%20where%20documentID%3D$filehead" | grep -A 2 ">entryUpdated"| tail -n 1 | sed 's/.*>\([0-9][-0-9 :-]*\).*/\1/') + db_epoch=$(date --date "$dbdate" +"%s") + diff=$(($file_epoch - $db_epoch)) + diff_min=$(($diff / 60)) + + a=$(invalid_diff $diff) + + if [ "$a" == 1 ]; then + echo "$curr_epoch indexer_running yes -1 $filehead $dbhead" + else + echo "$curr_epoch indexer_running yes $diff_min $filehead $dbhead" + fi +elif [ "$indexer_running" == -1 ]; then + echo "$curr_epoch indexer_running no $diff_min $filehead $dbhead" +elif [ "$indexer_running" == -2 ]; then + echo "$curr_epoch indexer_running NO $diff_min $filehead $dbhead" +else + echo "indexer_running NO -1 $filehead $dbhead UNKNOWN" +fi + diff --git a/usr/lib/periodic-check/script/site_length b/usr/lib/periodic-check/script/site_length new file mode 100755 index 00000000..fcc0d870 --- /dev/null +++ b/usr/lib/periodic-check/script/site_length @@ -0,0 +1,35 @@ +#!/bin/bash + +sites="localhost" +tomcat_port=8040 +newline=" +" +this_script=$(basename $0) + +function init { + . "$(dirname $0)/../main.conf" + if [ -f "$(dirname $0)/../conf/$this_script.conf" ]; then + . "$(dirname $0)/../conf/$this_script.conf" + fi +} + +init + +for line in $sites; do + epoch=$(date +"%s") + before=$(date +"%s.%N") + size=$(curl --connect-timeout $connect_timeout --max-time $max_time -s http://localhost:$tomcat_port -H "Host:$line"| wc -c) + after=$(date +"%s.%N") + if [ "$size" == "" ]; then + size="0" + fi + time_took=$(echo "$after - $before"| bc | cut -c -5) + oneline=$(echo "$epoch $line $size $time_took") + if [ "$all_lines" == "" ]; then + all_lines="$oneline" + else + all_lines="$all_lines$newline$oneline" + fi +done + +echo "$all_lines" diff --git a/usr/local/src/unit-tests/common-test.sh b/usr/local/src/unit-tests/common-test.sh new file mode 100644 index 00000000..76de94c9 --- /dev/null +++ b/usr/local/src/unit-tests/common-test.sh @@ -0,0 +1,17 @@ +#! /usr/bin/env bash + +# by tkj@vizrt.com + +if [ ! -e bash_unit ]; then + wget --quiet https://raw.github.com/skybert/my-little-friends/master/bash/bash_unit +fi +if [ ! -e alexandria ]; then + wget --quiet https://raw.github.com/skybert/my-little-friends/master/bash/alexandria +fi + +source bash_unit + +function common_test_is_loaded() { + echo 1 +} + diff --git a/usr/local/src/unit-tests/test-ece-install-content-engine b/usr/local/src/unit-tests/test-ece-install-content-engine new file mode 100644 index 00000000..6537a253 --- /dev/null +++ b/usr/local/src/unit-tests/test-ece-install-content-engine @@ -0,0 +1,32 @@ +#! /usr/bin/env bash + +# by tkj@vizrt.com + +common_test_is_loaded > /dev/null 2>&1 || source common-test.sh + +# source ../../../share/escenic/ece-scripts/ece-install.d/constants.sh +source ../../../share/escenic/ece-scripts/ece-install.d/content-engine.sh + +function test_get_publication_short_name_list() { + escenic_root_dir=$(mktemp -d) + local pub_dir=$escenic_root_dir/assemblytool/publications + mkdir -p $pub_dir + touch $pub_dir/{pub1,pub2}.properties + + # a sub dir + mkdir -p $pub_dir/sub + touch $pub_dir/sub/{SomeNurseryComponent,SomeOtherNurseryComponent}.properties + + local short_name_list=$(get_publication_short_name_list | sed 's/\ //g') + assert_equals "should not include other .properties" "pub2pub1" $short_name_list + + rm -rf $escenic_root_dir +} + +test_suite=" +test_get_publication_short_name_list +" + +run_test_suite $test_suite + + diff --git a/usr/sbin/drop-and-create-ecedb b/usr/sbin/drop-and-create-ecedb deleted file mode 100644 index de52738c..00000000 --- a/usr/sbin/drop-and-create-ecedb +++ /dev/null @@ -1,123 +0,0 @@ -#! /usr/bin/env bash - -# Script that will drop and re-create an ECE database, running -# all ECE and ECE plugins SQL scripts in the correct order. -# -# Currently, the script supports the following DBs: -# * Oracle (when prompted by SQLPlus for io_owner, enter your $user, -# for io_tablespace enter your $tablespace_data and for index -# table space your $tablespace_index). -# * MySQL -# -# Enjoy! -# -# -torstein@escenic.com - -drop_db_first=0 -user=ece5user -password=ece5password -db=ece5db -host=localhost -ece_home=/opt/escenic/engine -dbproduct=mysql -id="[`basename $0`]" - -# oracle specific settings -create_oracle_user=0 -tablespace_data=ece5_data -tablespace_index=ece5_index -oracle_data_dir=/home/oracle/app/oracle/oradata/orcl - -function create_oracle_ece_user() { - sqlplus /nolog << EOF - connect /as sysdba; - create user $user - identified by $password - default tablespace $tablespace_data - quota unlimited on $tablespace_data; - grant connect to $user; - grant resource to $user; - grant create any view to $user; - grant execute on ctx_ddl to $user; -EOF -} - -function run_db_scripts() -{ - for el in $db_fn_list; do - file=$1/$el.sql - echo $id "running $file ..." - if [ -e $1/$el.sql ]; then - if [ $dbproduct = "oracle" ]; then - sqlplus $user/$password @$file - else - mysql -u $user -p$password -h $host $db < $file - fi - fi - done -} - -if [ $create_oracle_user -eq 1 ]; then - create_oracle_ece_user -fi - -if [ $drop_db_first -eq 1 ]; then - echo $id "dropping and re-creating $db on $host ..." - if [ $dbproduct = "mysql" ]; then - mysql -h $host << EOF - drop database $db; -EOF - else - sqlplus /nolog << EOF - connect /as sysdba; - drop tablespace $tablespace_data including contents; - drop tablespace $tablespace_index including contents; -EOF - fi -fi - -# we first create the DB (or, if drop_db_first is 1, we've just -# dropped it above) before running the SQL scripts. -if [ $dbproduct = "mysql" ]; then - mysql -h $host << EOF - create database $db character set utf8 collate utf8_general_ci; - grant all on $db.* to $user@'%' identified by '$password'; - grant all on $db.* to $user@'localhost' identified by '$password'; -EOF -else - sqlplus /nolog << EOF - connect /as sysdba; - - create tablespace $tablespace_data - datafile '$oracle_data_dir/${tablespace_data}01.dbf' - size 200M reuse - autoextend on next 50M maxsize 2047M - extent management local autoallocate; - - create tablespace $tablespace_index - datafile '$oracle_data_dir/${tablespace_index}01.dbf' - size 100M reuse - autoextend on next 50M maxsize 2047M - extent management local autoallocate; -EOF -fi - - -db_fn_list=" -tables -tables-stats -views -constants -constants-stats -constraints -indexes -history -" - -run_db_scripts $ece_home/database/$dbproduct - -for el in `find -L $ece_home/plugins -name $dbproduct`; do - run_db_scripts $el -done - -echo "${id} ${dbproduct}://${host}/${db} is now ready for ${user}/${password}" diff --git a/usr/sbin/ece-deploy b/usr/sbin/ece-deploy new file mode 100755 index 00000000..a790bddc --- /dev/null +++ b/usr/sbin/ece-deploy @@ -0,0 +1,609 @@ +#! /usr/bin/env bash + +## Command which deployes an EAR and DEB package by their URIs. +## +## The command will deploy the EAR for all Escenic Content Engines and +## Search servers installed on the given host and will figure out +## instances and user/passwords by itself. It will also clear the work +## directory before starting the instances again. The last thing the +## script does, is to deploy the conf package. + +ece_deploy_data_dir=/var/lib/escenic/$(basename $0) +ece_deploy_download_dir=/var/cache/escenic +log=/var/log/escenic/$(basename $0).log + +function ece_deploy_cancel_hook() { + local the_dir=$ece_deploy_data_dir/$deployment_id + run rm -rf $the_dir + common_bashing_user_cancelled_hook +} + +function create_pretty_printed_file_if_xml() { + if [ ! $1 ]; then + return + fi + if [ ! -e $1 ]; then + return + fi + + if [ $(looks_like_xml ${1}) -eq 1 ]; then + xml_pp ${1} | xmllint --noblanks --format - > ${1}.xml + fi +} + +function bootstrap_thyself() { + # first, try to be nice, then check the standard location + local dir=$(dirname $0)/../share/escenic/ece-scripts + if [ ! -d $dir ]; then + dir=/usr/share/escenic/ece-scripts + fi + + local common_libraries=" + common-bashing.sh + common-io.sh + common-os.sh + common-ece.sh + " + + for el in $common_libraries; do + source $dir/$el 2>/dev/null || { + echo "$(basename $0): Could not load the library $el," \ + "and I can't live without it :-(" | fmt + exit 1 + } + done + + create_pid + create_lock + + # hooks for when the scirpt exits cleanly and when a user or someone + # kills the process + trap common_bashing_exit_hook EXIT + trap ece_deploy_cancel_hook SIGINT SIGHUP + + source /etc/default/ece 2>/dev/null || { + print "There's no /etc/default/ece on $HOSTNAME" \ + "this means there's no ece-installed ECE here, I will exit" + exit 1 + } +} + +bootstrap_thyself + +function deploy_ear() { + local proxy_setings="" + if [ -n "$http_proxy" ]; then + proxy_settings="http_proxy=$http_proxy" + fi + + for el in $engine_instance_list; do + su - $ece_unix_user -c \ + "${proxy_settings} ece -i $el stop deploy clean start --uri $ear_uri" + if [ ${PIPESTATUS[@]} -gt 0 ]; then + exit 1 + fi + done + + for el in $search_instance_list; do + su - $ece_unix_user -c \ + "${proxy_settings} ece -i $el -t search stop deploy clean start --uri $ear_uri" + exit_on_error "Running ece deploy as $ece_unix_user" + done +} + +function start_all_content_engine_instances() { + for el in $engine_instance_list; do + su - $ece_unix_user -c \ + "${proxy_settings} ece -i $el restart" + done +} + +function set_builder_user_and_password() { + run source /etc/escenic/ece.conf + for el in $engine_instance_list; do + file=/etc/escenic/ece-${el}.conf + if [ -e $file ]; then + run source $file + fi + done + + local found=0 + if [[ -n "$builder_http_user" && -n "$builder_http_password" ]]; then + found=1 + fi + + if [ $found -eq 0 ]; then + local file_list=/etc/escenic/ece.conf + for el in $engine_instance_list; do + file_list="$file_list /etc/escenic/ece-${el}.conf" + done + print_and_log \ + "You haven't set builder_http_user & and builder_http_password" \ + "in neither of these files:" \ + $file_list \ + "I assume the downloads are not password protected" + fi +} + +function deploy_conf() { + if [ ${make_deployment-1} -eq 0 ]; then + return + fi + + make_dir $ece_deploy_download_dir + + set_builder_user_and_password + + print "Downloading" $conf_package_uri "..." + + # common-ece::download_uri_target_to_dir honors wget_auth + wget_auth=" + --http-user $builder_http_user + --http-password $builder_http_password + " + download_uri_target_to_dir $conf_package_uri $ece_deploy_download_dir + + print "Installing" $(basename $conf_package_uri) "..." + if [ ${force_everything-0} -eq 1 ]; then + dpkg_opts="--force-overwrite --force-confnew" + print_and_log $(yellow WARNING) "using force as you requested" \ + "installing" $(basename $conf_package_uri) "using these flags:" \ + $dpkg_opts + fi + + dpkg \ + $dpkg_opts \ + --install $ece_deploy_download_dir/$(basename $conf_package_uri) +} + +function read_user_input() { + local next_is_ear_uri=0 + local next_is_conf_uri=0 + + for el in "$@"; do + if [[ $el == "--ear" ]]; then + deploy_an_ear_file=1 + next_is_ear_uri=1 + elif [[ $el == "--conf" ]]; then + deploy_a_conf_file=1 + next_is_conf_uri=1 + elif [[ $el == "--rollback" ]]; then + roll_back=1 + next_is_roll_back_to_version=1 + make_deployment=0 + elif [[ $el == "--force" ]]; then + force_everything=1 + elif [[ $el == "--update-publication-resources" ]]; then + update_the_publication_resources=1 + elif [[ $el == "--list-deployments" ]]; then + list_the_revious_deployments=1 + make_deployment=0 + elif [ $next_is_ear_uri -eq 1 ]; then + ear_uri=$el + next_is_ear_uri=0 + elif [ $next_is_conf_uri -eq 1 ]; then + conf_package_uri=$el + next_is_ear_uri=0 + elif [ ${next_is_roll_back_to_version-0} -eq 1 ]; then + roll_back_to_version=$el + next_is_roll_back_to_version=0 + fi + done +} + +function verify_user_input() { + local errors=0 + if [ ${roll_back-0} -eq 1 ]; then + if [ -z "${roll_back_to_version}" ]; then + print "You must specify which version to roll back to." + errors=1 + fi + elif [ ${list_the_revious_deployments-0} -eq 1 ]; then + nop=foo + fi + + if [[ ${deploy_an_ear_file-0} -eq 1 && -z "${ear_uri}" ]]; then + print "You must specify the URI of the EAR file with --ear :-(" + errors=1 + fi + + if [[ ${deploy_a_conf_file-0} -eq 1 && -z "$conf_package_uri" ]]; then + print "You must specify the conf package with --conf :-(" + errors=1 + fi + + if [[ ${roll_back-0} -eq 0 && \ + ${list_the_revious_deployments-0} -eq 0 && \ + ${deploy_an_ear_file-0} -eq 0 && \ + ${deploy_a_conf_file-0} -eq 0 ]]; then + errors=1 + fi + + if [ $errors -eq 1 ]; then + exit 1 + fi +} + +function list_previous_deployments() { + if [ ! -d $ece_deploy_data_dir ]; then + return + fi + + for el in $(find $ece_deploy_data_dir -maxdepth 1 -type d | \ + sed "s#${ece_deploy_data_dir}##g" | \ + sort -n | \ + tail -20); do + local id=$(basename $el) + echo " - Deployment" ${id} \ + "was made @" \ + $(date --date="@$(echo ${id} | cut -d'-' -f2)") + done +} + +## $1 :: file that you want to see if it's XML or not +function looks_like_xml() { + if [ -z "$1" ]; then + echo 0 + fi + + if [ ! -r "$1" ]; then + echo 0 + fi + + local xml=$(cat "$1") + if [[ ${xml:0:1} == "<" ]]; then + echo 1 + else + echo 0 + fi +} + +function get_deployment_dir() { + echo $ece_deploy_data_dir/$deployment_id +} + +function get_publication_new_dir() { + echo $(get_deployment_dir)/new/$publication +} + +function get_publication_old_dir() { + echo $(get_deployment_dir)/old/$publication +} + +## $1 :: base dir of the new publication resource to apply to the +## running instance. If this base dir is the $tomcat_base from the +## ece-.conf, the function will figure out the path to the +## publication resources inside the exploded publication specific +## webapps directories by itself. +function update_publication_resources() { + if [ ! -d $1 ]; then + return + fi + + print_and_log "Will now update the publication resources," \ + "first getting a list of all publications on $el ..." + for i in {0..5}; do + local publication_list=$(get_publication_list ${appserver_port}) + + if [ -n "${publication_list}" ]; then + break + fi + + # if we haven't gotten any content from the local ECE (yet), it + # might be because it hasn't come up yet (slow app server), + # hence we sleep for a wee while and try again. + sleep 3 + done + + if [ -z "${publication_list}" ]; then + print_and_log "$el doesn't have any publications" \ + "so there's no publication resources to update ;-)" + continue + else + print_and_log "${el}@${HOSTNAME} has the following publications:" \ + $publication_list + fi + + for publication in $publication_list; do + # this means a regular deployment + if [[ "$1" == "$tomcat_base" ]]; then + local publication_resources_dir="/tmp/no/thing" + # must search for the right context in defaults.properties as + # the WAR file can differ from the publication name. + for el in $(find $tomcat_base/webapps-* \ + -name defaults.properties 2>/dev/null); do + if [ $(grep "^publication-name=${publication}$" $el | wc -l) -gt 0 ]; then + local base_dir=${el%/WEB-INF/localconfig/defaults.properties} + publication_resources_dir=${base_dir}/META-INF/escenic/publication-resources + # we're happy, so break out of the loop + break + fi + done + # this means a roll back + else + local publication_resources_dir=${1}/${publication} + fi + + if [ ! -d $publication_resources_dir ]; then + print_and_log "Couldn't find the publication resource dir for" \ + "publication" $publication ", I will not update its resources." + continue + fi + + print_and_log "Updating resources for publication" $publication "..." + # we grep away .xml and .diff file shere as these will be present + # when rolling back a deployment. + for publication_resource in \ + $(find $publication_resources_dir -type f | egrep -v ".(xml|diff)$"); do + local relative_resource=$( + echo $publication_resource | sed "s#${publication_resources_dir}/##g" + ) + + local new_file=$(get_publication_new_dir)/${relative_resource} + local old_file=$(get_publication_old_dir)/${relative_resource} + make_dir $(dirname $new_file) $(dirname $old_file) + + # copy the new file to the deployment data directory + run cp $publication_resource $new_file + + # when updating the resources as a part of a regular deployment + # (as opposed to a roll back) we must download the old/current + # version of the file. + if [ ! -e $old_file ]; then + wget \ + --quiet \ + --continue \ + --output-document $old_file \ + http://localhost:${appserver_port}/escenic-admin/publication-resources/$publication/${relative_resource} \ + 2>/dev/null + fi + + create_pretty_printed_file_if_xml $old_file + create_pretty_printed_file_if_xml $new_file + + local diff_from_file=$old_file + local diff_to_file=$new_file + + if [[ -e ${old_file}.xml && -e ${new_file}.xml ]]; then + diff_from_file=$old_file.xml + diff_to_file=$new_file.xml + fi + + # in case the running system doesn't have the publication + # resource from the the new EAR. + if [ ! -e $old_file ]; then + old_file=$(mktemp) + fi + + diff -w ${diff_from_file-$old_file} \ + ${diff_to_file-$new_file} \ + > ${new_file}.diff + local number_of_lines_changed=$(egrep '<|>' ${new_file}.diff | wc -l) + if [ $number_of_lines_changed -lt 1 ]; then + continue + fi + + print_and_log "Applying" $number_of_lines_changed \ + "changes to" $publication_resource + run curl \ + --silent \ + --upload-file $new_file \ + --fail \ + --verbose \ + http://localhost:${appserver_port}/escenic-admin/publication-resources/${publication}/${relative_resource} + done + done +} + +function generate_deployment_id() { + deployment_id=${HOSTNAME}-$(date +%s) + print_and_log "This deployment has ID" ${deployment_id} \ + "and is logging to" $log +} + +function roll_back_everything_to_version() { + print_and_log "Rolling back to" \ + ${roll_back_to_version} \ + "from" $(date --date @$(echo ${roll_back_to_version} | cut -d'-' -f2)) \ + "..." + + local new_ece_deploy_data_dir=$ece_deploy_data_dir/${roll_back_to_version}/new + if [ ! -d $new_ece_deploy_data_dir ]; then + print_and_log "I cannot find ${roll_back_to_version}'s files," \ + "they should have been here:" $new_ece_deploy_data_dir + exit 1 + else + print_and_log "Using data in" $new_ece_deploy_data_dir "to perform the rollback" + fi + + local package_version_file=$new_ece_deploy_data_dir/vosa-conf-${HOSTNAME}.version + if [ -e $package_version_file ]; then + local deb_file=$ece_deploy_download_dir/vosa-conf-$HOSTNAME-$(cat $package_version_file).deb + if [ -e $deb_file ]; then + print_and_log "Reverting back to version" $(cat $package_version_file) \ + "vosa-conf-$HOSTNAME ..." + dpkg --install ${dpkg_opts} $deb_file + else + print_and_log "You must now install version" \ + $(cat $package_version_file) \ + "of the vosa-conf-$HOSTNAME package" \ + "I couldn't find it in" $deb_file + fi + else + print_and_log $(yellow WARNING) \ + "Couldn't find the previous version for vosa-conf-$HOSTNAME" \ + "you must get a hold of the latest/approriate one yourself." + fi + + for el in $engine_instance_list; do + local file=$new_ece_deploy_data_dir/${el}.state + if [ ! -e $file ]; then + continue + fi + + local old_ear=$(grep ear_used $file | cut -d'=' -f2) + print_and_log "Rolling back ${el}'s EAR to ${old_ear} ..." + ear_uri=${old_ear} + deploy_ear + done +} + +## $1 :: dir, can be the path to the new or old directory of this +## deployment +function remember_state() { + make_dir $1 + + print_and_log "Remembering the deployment states for all instances ..." + for el in $engine_instance_list $search_instance_list; do + local file=/var/lib/escenic/${el}.state + if [ -e $file ]; then + cp /var/lib/escenic/${el}.state $1/ + fi + done + + if [ $(dpkg -l vosa-conf-$HOSTNAME 2>/dev/null | wc -l) -eq 0 ]; then + return + fi + + dpkg -l vosa-conf-$HOSTNAME | \ + grep vosa-conf-$HOSTNAME | \ + awk '{ print $3 }' > \ + $1/vosa-conf-${HOSTNAME}.version + + print_and_log "Remembering all files from the current version of" \ + vosa-conf-$HOSTNAME "..." + dpkg -L vosa-conf-$HOSTNAME | while read f; do + local dir=${1}/vosa-conf-${HOSTNAME}/$(dirname $f) + make_dir $dir + if [ -d $f ]; then + continue + fi + if [ ! -e $f ]; then + print_and_log $(yellow WARNING) $f "was installed with the" \ + "vosa-conf-$HOSTNAME package," \ + "however, it's not present on $HOSTNAME." + continue + fi + cp $f $dir/ + done +} + +function print_summary() { + for el in $engine_instance_list $search_instance_list; do + for ele in ${el}.state vosa-conf-$HOSTNAME.version; do + if [[ -e $(get_deployment_dir)/old/${ele} && \ + -e $(get_deployment_dir)/new/${ele} ]]; then + diff --ignore-all-space $(get_deployment_dir)/old/${ele} \ + $(get_deployment_dir)/new/${ele} \ + > $(get_deployment_dir)/new/${ele}.diff + fi + done + done + + if [[ $(dpkg -l vosa-conf-$HOSTNAME 2>/dev/null | wc -l) -gt 0 && \ + -d $(get_deployment_dir)/old/vosa-conf-$HOSTNAME && \ + -d $(get_deployment_dir)/new/vosa-conf-$HOSTNAME ]]; then + diff --recursive --ignore-all-space \ + $(get_deployment_dir)/old/vosa-conf-$HOSTNAME \ + $(get_deployment_dir)/new/vosa-conf-$HOSTNAME \ + > $(get_deployment_dir)/new/vosa-conf-${HOSTNAME}.diff + fi + + local diff_files=$( + find $(get_deployment_dir) -name "*.diff" | \ + grep -v 'usr/share/doc' + ) + if [ $(echo "$diff_files" | wc -c) -gt 2 ]; then + print_and_log "Here are the diffs of all publication resources" \ + "and files that were changed:" + for el in $diff_files; do + if [ $(wc -c $el | awk '{print $1;}') -gt 0 ]; then + echo $el $(egrep '<|>' $el | wc -l) "changes" + fi + done + fi + + print_and_log "Finished @ $(date)" +} + +function crawl_a_wee_bit() { + for el in $engine_instance_list; do + source /etc/escenic/ece-${el}.conf + local file=$tomcat_base/conf/server.xml + local virtual_hosts=$( + xml_grep \ + --nowrap \ + --text \ + --cond 'Server/Service/Engine/Host/Alias' \ + $file + ) + + for ele in $virtual_hosts; do + print_and_log "Crawling the front page (including JS, CSS, pictures) " \ + "of $ele ..." + wget \ + --quiet \ + --page-requisites \ + --delete-after \ + --header "Host: $ele" http://localhost:${appserver_port}/ + done + done +} + +assert_commands_available xmllint xml_grep xml_pp +read_user_input "$@" +verify_user_input + +if [ ${list_the_revious_deployments-0} -eq 0 ]; then + print_and_log "Started @ $(date)" + generate_deployment_id +fi + +if [ ${make_deployment-1} -eq 1 ]; then + make_dir $(get_deployment_dir)/old $(get_deployment_dir)/new + remember_state $(get_deployment_dir)/old + + if [ -n "$ear_uri" ]; then + deploy_ear + fi + + if [ -n "$conf_package_uri" ]; then + deploy_conf + fi + + if [ ${update_the_publication_resources-0} -eq 1 ]; then + for el in $engine_instance_list; do + run source /etc/escenic/ece-${el}.conf + update_publication_resources $tomcat_base + done + fi + + start_all_content_engine_instances + remember_state $(get_deployment_dir)/new + crawl_a_wee_bit +fi + +if [ ${list_the_revious_deployments-0} -eq 1 ]; then + list_previous_deployments +fi + +if [ ${roll_back-0} -eq 1 ]; then + make_dir $(get_deployment_dir)/old $(get_deployment_dir)/new + remember_state $(get_deployment_dir)/old + + roll_back_everything_to_version + + if [ ${update_the_publication_resources-0} -eq 1 ]; then + for el in $engine_instance_list; do + run source /etc/escenic/ece-${el}.conf + update_publication_resources $ece_deploy_data_dir/${roll_back_to_version}/new + done + fi + + remember_state $(get_deployment_dir)/new + print_and_log "You have now rolled back to deployment ${roll_back_to_version}" +fi + +if [ ${list_the_revious_deployments-0} -eq 0 ]; then + print_summary +fi diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install old mode 100644 new mode 100755 index 93279362..c5a11984 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -1,479 +1,302 @@ #! /usr/bin/env bash -# The goal of this script, is to have an ECE up and running within -# five minutes. The setup the user is getting is suitable for a -# production environment, except for the fact that with this script, -# all ECE components are set up on the same host. +# The goal of this script is install a complete production environment +# for web sites using on Escenic Content Engine as their CMS. The +# script is equally well suited for installing development, testing +# and staging environments, in addition to recover from backups create +# with the "ece -i backup" command. +# +# Always check for the latest version at +# http://github.com/vizrt/ece-scripts -# by tkj@vizrt.com +# echo comments and suggestions > tkj@vizrt.com +# echo frustrations > /dev/null ##################################################################### -# User definable variables (the defaults are fine in most cases). +# User definable variables (the defaults are fine in most +# cases). These are the most likely variables you want to change and +# they can all be set in the ece-install.conf file ##################################################################### ece_user=escenic ece_group=escenic -jdbc_driver=/usr/share/java/mysql.jar + +#Specify the jdbc_driver variable if you want to use something else +#than the default. For MySQL and Percona the default is +#/usr/share/java/mysql-connector-java.jar from the libmysql-java +#package. For MariaDB the driver is downloaded from +#downloads.mariadb.org. +#jdbc_driver=/usr/share/java/mysql-connector-java.jar debug=0 -# don't touch this one if you're installing on a Debian based system -# as it'll be set up for you by the script itself. -java_home=/usr/lib/jvm/java-6-sun +# These variables govern where software is installed and data and run +# time files are written. +dir_suffix=escenic +escenic_root_dir=/opt/${dir_suffix} +escenic_conf_dir=/etc/${dir_suffix} +escenic_log_dir=/var/log/${dir_suffix} +escenic_data_dir=/var/lib/${dir_suffix} +escenic_run_dir=/var/run/${dir_suffix} +escenic_backups_dir=/var/backups/${dir_suffix} +escenic_spool_dir=/var/spool/${dir_suffix} +escenic_cache_dir=/var/cache/${dir_suffix} +escenic_crash_dir=/var/crash/${dir_suffix} +appserver_parent_dir=/opt + +# country code for selecting the correct (APT) mirror. +mirror_country_suffix=no + +java_package_name="" + +# This variable is updated by the package build scripts. +ece_scripts_version="straight-from-github" + +#Set the default database vendor. Possible values are mariadb and percona +db_vendor=percona + ##################################################################### -id="[$(basename $0)]" -pid_file=/var/run/$(basename $0).pid -download_dir=/tmp/ece-downloads -log=/tmp/$(basename $0).log -conf_file=$HOME/.ece-install.conf -common_nursery_dir=/etc/escenic/engine/common +download_dir=${escenic_cache_dir}/$(basename $0) +log=${escenic_log_dir}/$(basename $0).log +conf_file=$HOME/ece-install.conf +ece_scripts_git_source=https://github.com/vizrt/ece-scripts.git +maven_opts="--batch-mode" +wget_opts="--continue --inet4-only --quiet" +apt_opts="--no-install-recommends" +curl_opts="--silent" + +# hook scripts +ece_install_scripts_dir=$HOME/$(basename $0).d # globals will be set to correct values in run-time. +appserver_host=localhost appserver_port=8080 on_debian_or_derivative=0 +on_redhat_or_derivative=0 on_debian=0 +on_redhat=0 +on_ubuntu=0 +force_packages=0 # because the all in one profile will run database, search and app # server profiles, all of which needs downloading and setting up the # ECE software components. ece_software_setup_completed=0 -NEW_LINE=" -" -# the next steps printed when the user has installed his/her -# components. -next_steps="" - -function debug() -{ - if [ $debug -eq 1 ]; then - echo "[$(basename $0)-debug]" "$@" - fi -} - -function print() -{ - echo $id $@ -} - +############################################################################ +# For ECE 5.5 +############################################################################ technet_download_list=" -http://technet.escenic.com/downloads/assemblytool-2.0.2.zip -http://technet.escenic.com/downloads/release/53/analysis-engine-2.3.6.0.zip -http://technet.escenic.com/downloads/release/53/community-engine-3.6.1.0.zip -http://technet.escenic.com/downloads/release/53/dashboard-1.0.0.0.zip -http://technet.escenic.com/downloads/release/53/engine-5.3.2.2.zip -http://technet.escenic.com/downloads/release/53/forum-3.0.0.0.zip -http://technet.escenic.com/downloads/release/53/inpage-1.3.0.0.zip -http://technet.escenic.com/downloads/release/53/lucy-dist-4.1.6.0.zip -http://technet.escenic.com/downloads/release/53/menu-editor-dist-2.0.6.0.zip -http://technet.escenic.com/downloads/release/53/poll-2.1.3.0.zip -http://technet.escenic.com/downloads/release/53/xml-editor-dist-2.1.0.0.zip + http://technet.escenic.com/downloads/release/55/analysis-engine-2.5.0.131590.zip + http://technet.escenic.com/downloads/release/55/assemblytool-2.0.4.jar + http://technet.escenic.com/downloads/release/55/community-engine-3.8.1.132399.zip + http://technet.escenic.com/downloads/release/55/engine-5.5.0.131978.zip + http://technet.escenic.com/downloads/release/55/forum-3.2.1.132293.zip + http://technet.escenic.com/downloads/release/55/geocode-2.4.1.130693.zip + http://technet.escenic.com/downloads/release/55/lucy-4.2.1.129603.zip + http://technet.escenic.com/downloads/release/55/menu-editor-2.1.1.128528.zip + http://technet.escenic.com/downloads/release/55/online-graphics-1.0.1.133123.zip + http://technet.escenic.com/downloads/release/55/poll-2.3.0.128547.zip + http://technet.escenic.com/downloads/release/55/vcpeditor-1.0.0.132434.zip + http://technet.escenic.com/downloads/release/55/video-1.2.1.132710.zip " - wf_download_list=" -http://technet.escenic.com/downloads/widget-framework/widget-framework-core-1.10.0.0.zip + http://technet.escenic.com/downloads/release/55/framework-dist-2.0.2.131581.zip + http://technet.escenic.com/downloads/release/55/framework-community-dist-2.0.2.131581.zip " -tomcat_download=http://ftp.nsysu.edu.tw/Apache/tomcat/tomcat-6/v6.0.32/bin/apache-tomcat-6.0.32.tar.gz - -PROFILE_ALL_IN_ONE=1 -PROFILE_CACHE_SERVER=5 -PROFILE_DB_SERVER=4 -PROFILE_EDITORIAL_SERVER=2 -PROFILE_PRESENTATION_SERVER=3 -PROFILE_RMI_HUB=6 -PROFILE_SEARCH_SERVER=7 -PROFILE_WIDGET_FRAMEWORK=8 -PROFILE_CREATE_PUBLICATION=9 - -# Because of issue VF-3559, we also create the default family and host -# directories. -dir_list=" -$common_nursery_dir -/etc/escenic/engine/family/default -/etc/escenic/engine/host/localhost -/opt/escenic -/var/cache/escenic -/var/crash/escenic -/var/lib/escenic -/var/log/escenic -/var/run/escenic -/var/spool/escenic/migration -" - -function make_dir() -{ - if [ ! -d $1 ]; then - mkdir -p $1 - fi -} - -for el in $dir_list; do - make_dir $el -done +############################################################################ + +# sources.list used by ece-install +escenic_sources=/etc/apt/sources.list.d/escenic.list + +# if set to 1, ece-install will try do as much dry running as +# possible. I.e. not actually install things, but only download and +# prepare the pre-requisites. +dry_run=0 + +## Bootstrapping, load files from /usr/share/escenic/ece-scripts The +## method will first try to be smart, in case the user has copied the +## ece-scripts somewhere else., e.g.: moved everything to ~/ece-scrpts +## or /opt/escenic/ece-scripts, this should also work. +function init() { + # before doing anything else, be sure that the directory of the log + # file exists. + local dir=$(dirname $log) + if [ ! -d $dir ]; then + mkdir $(dirname $log) 2>/dev/null || { + echo "Couldn't create $(dirname $log)" + exit 1 + } + fi + + # first, try to be nice + local dir=$(dirname $0)/../share/escenic/ece-scripts + + # then check the standard location + if [ ! -d $dir ]; then + dir=/usr/share/escenic/ece-scripts + fi + + if [ -d $dir ]; then + # load common librariees + common_libraries="common-ece.sh common-bashing.sh common-io.sh common-os.sh" + for el in $common_libraries; do + source $dir/${el} || exit $? + done -function make_ln() -{ - if [ -e $1 -a ! -h $(basename $1) ]; then - ln -s $1 - elif [ ! -e $1 ]; then - print "Tried to make a symlink to $1, but it doesn't exist" - exit 1 - fi + # load ece-install modules + for el in $dir/ece-install.d/*.sh; do + log "Loading $(basename $0) module:" $(basename $el) + source $el || exit $? + done + else + echo "I cannot find $(basename $0)'s dependencies, exiting :-(" + exit 1 + fi + + # hooks for when the scirpt exits cleanly and when a user or someone + # kills the process + trap common_bashing_exit_hook EXIT + trap common_bashing_user_cancelled_hook SIGINT SIGHUP } -# TODO download documentation to -# /usr/share/doc/escenic/content-engine-/ - function download_escenic_components() { if [ $ece_software_setup_completed -eq 1 ]; then return fi - - print "Downloading Escenic software from technet.escenic.com ..." - - cd $download_dir - for el in $technet_download_list; do - if [ -e $(basename $el) ]; then - continue - fi - - wget --continue \ - --http-user $technet_user \ - --http-password $technet_password \ - $el \ - 1>>$log 2>>$log - done - -} - -# will intall the passed packages if these are not installed from -# before. -# -# parameters: -# $1 : space separated string of package names -install_packages_if_missing() -{ - if [ $on_debian_or_derivative ]; then - some_are_missing=0 - for el in $@; do - # we don't need to grep away "No packages found matching - # ..." since this message from dpkg is written to standard - # error. - if [ $(dpkg -l $el 2>/dev/null | grep ^ii | wc -l) -lt 1 ]; then - some_are_missing=1 - fi - done - - if [ $some_are_missing -eq 0 ]; then - return - fi - - apt-get install -y $@ 1>>$log 2>>$log - fi -} - - -function install_common_os_packages() -{ - print "Installing 3rd party packages needed by $(basename $0) ..." - - if [ $on_debian_or_derivative -eq 1 ]; then - # Ubuntu doesn't have git (!) but only git-core. - if [ $on_ubuntu -eq 1 ]; then - git_package=git-core - else - git_package=git - fi - - packages="curl $git_package wget" - install_packages_if_missing $packages - fi - assert_pre_prequesite curl - assert_pre_prequesite wget - assert_pre_prequesite git -} + print_and_log "Downloading software from technet.escenic.com ..." + run cd $download_dir -make_dir $ece_directories + for el in $technet_download_list $wf_download_list; do + if [ -e $(basename $el) ]; then + continue + fi -function set_up_assembly_tool() -{ - print "Setting up the Assembly Tool ..." - - make_dir /opt/escenic/assemblytool/ - cd /opt/escenic/assemblytool/ - - if [ -e $download_dir/assemblytool*zip ]; then - unzip -u $download_dir/assemblytool*zip \ - 1>>$log 2>>$log - mv $download_dir/assemblytool*zip /tmp/ - fi + log "Downloading $el ..." + run wget $wget_opts \ + --http-user $technet_user \ + --http-password $technet_password \ + $el + done - # adding an instance layer to the Nursery configuration - cp -r /opt/escenic/engine/siteconfig/bootstrap-skeleton \ - /opt/escenic/assemblytool/conf - cd /opt/escenic/assemblytool/conf/ - cp -r layers/host layers/instance - cat > layers/instance/Files.properties <> Nursery.properties - echo "layer.06 = /layers/instance/Layer" >> Nursery.properties - echo "come here" > /tmp/tkj - - # set up which plugins to use - cd /opt/escenic/assemblytool/ - make_dir plugins - cd plugins - find ../../ -maxdepth 1 -type d | \ - grep -v assemblytool | \ - while read directory; do - if [ $directory = "../../" -o \ - $(echo $directory | grep widget-framework | wc -l) -gt 0 ]; then - continue + for el in $ear_download_list ; do + if [ -s $(basename $el) ]; then + continue; + fi + local repo + local targetname=$(basename $el) + for repo in $fai_maven_repositories; do + # Special handling for snapshots + # It will try to get the maven-metadata file. If the file exist it will create the proper el and download it later. VOSA-108 + if curl -s -I $repo/$(dirname $el)/maven-metadata.xml | grep -q '^HTTP[^ ]* 200' ; then + local metadata; + local name=$(basename $el) + metadata=$(curl -s $repo/$(dirname $el)/maven-metadata.xml) + local timestamp=$(xmlstarlet <<< "$metadata" sel -t -m /metadata/versioning/snapshot/timestamp -c 'text()') + local buildnumber=$(xmlstarlet <<< "$metadata" sel -t -m /metadata/versioning/snapshot/buildNumber -c 'text()') + #Now Check the the timestamp and buildnumber is not empty. If not then create the dist name with timestamp and buildnumber. + if [[ -n $timestamp && -n $buildnumber ]]; then + log "Found SNAPSHOT with $timestamp-$buildnumber in $el" + el=$(dirname $el)/${name/SNAPSHOT/$timestamp-$buildnumber} + fi + log "Downloading $el from $repo". fi - - # nuisance to get the community engine, but not the engine - if [ $(echo $directory | grep engine | wc -l) -gt 0 ]; then - if [ $(echo $directory | grep community | wc -l) -lt 1 ]; then - continue - fi + if curl -s -I $repo/$el | grep -q '^HTTP[^ ]* 200' ; then + log "Downloading $(basename $el) from $repo/$el ..." + run wget $wget_opts $repo/$el -O $targetname + break + else + log "$(basename $el) not found using $repo/$el" fi - - make_ln $directory + done done - - cd /opt/escenic/assemblytool/ - ant -q initialize \ - 1>>$log 2>>$log - cat assemble.properties | \ - sed 's/#\ engine.root\ =\ \./engine.root=\/opt\/escenic\/engine/g' \ - > assemble.properties.tmp - mv assemble.properties.tmp assemble.properties - - cat assemble.properties | \ - sed 's/\#\# plugins\ =\ \/path\/to\/plugins/plugins=\/opt\/escenic\/assemblytool\/plugins/g' \ - > assemble.properties.tmp - mv assemble.properties.tmp assemble.properties } -function set_up_engine_and_plugins() -{ - if [ $ece_software_setup_completed -eq 1 ]; then - return - fi - - print "Setting up the Escenic Content Engine & its plugins ..." +# we need to do one apt-get update to be sure the package list is +# fresh. However, we don't want to do this in a common method, since +# it *might* be that all pre-requisite packages already are present. +one_time_apt_update_done=0 - make_dir /opt/escenic - cd /opt/escenic/ +# Will install the passed packages if these are not installed from +# before. +# +# parameters: +# $1 : space separated string of package names +function install_packages_if_missing() { + log "Installing package(s) [$@] if missing ..." - if [ ! -d engine-* ]; then - unzip -u $download_dir/engine*.zip \ - 1>>$log 2>>$log - if [ -h engine ]; then - rm engine - fi - - ln -s engine-* engine - mv $download_dir/engine*.zip /tmp - else - debug "engine- is already there, skipping to next step." - fi - - # now, there's only plugins left in the download dir - # we extract them in /opt/escenic as we want to re-use - # them between minor updates of ECE. - cd /opt/escenic/ - for el in $download_dir/*.zip; do - unzip -u $el \ - 1>>$log 2>>$log + if [ $on_debian_or_derivative -eq 1 ]; then + some_are_missing=0 + for el in $@; do + # we don't need to grep away "No packages found matching + # ..." since this message from dpkg is written to standard + # error. + if [ $(dpkg -l $el 2>/dev/null | grep ^ii | wc -l) -lt 1 ]; then + some_are_missing=1 + fi done - ece_software_setup_completed=1 -} - -function set_up_ece_scripts() -{ - print "Setting up the ece UNIX scripts ..." - - cd $download_dir - if [ -d ece-scripts ]; then - (cd ece-scripts - git pull 1>>$log 2>>$log) - else - git clone git://github.com/skybert/ece-scripts.git \ - 1>>$log 2>>$log - fi - - cp -r ece-scripts/usr/* /usr/ - cp -r ece-scripts/etc/* /etc/ - - # no need to add init.d scripts to the runlevel(s) for these - # profiles - if [ $install_profile_number -eq $PROFILE_WIDGET_FRAMEWORK -o \ - $install_profile_number -eq $PROFILE_SEARCH_SERVER -o \ - $install_profile_number -eq $PROFILE_DB_SERVER -o \ - $install_profile_number -eq $PROFILE_CACHE_SERVER ]; then - return + if [ $some_are_missing -eq 0 ]; then + return + elif [ $one_time_apt_update_done -eq 0 ]; then + log "First running APT update to ensure fresh package list, " \ + "then continuing the above" + run apt-get update + one_time_apt_update_done=1 fi - if [ $on_debian_or_derivative ]; then - print "Adding the ece init.d script to the default run levels..." - update-rc.d ece defaults \ - 1>>$log 2>>$log + if [ $force_packages -eq 1 ]; then + run apt-get install $apt_opts --assume-yes --force-yes $@ else - print "You remember to add /etc/intit.d/ece to the desired runlevels" - # TODO add init.d to the default runlevels, for other - # distributions too: - # - RedHat/chekcconfig - # - Gentoo: rc-update add ece default + run apt-get install $apt_opts --assume-yes $@ fi + elif [ $on_redhat_or_derivative -eq 1 ]; then + # since some versions of yum use --assume-yes and some use + # --assumeyes (!), we use the short form, -y, here + run yum install -y $@ + fi } -function set_up_ecedb() -{ - print "Setting up the ECE database schema ..." - - make_dir /opt/escenic/engine/plugins - cd /opt/escenic/engine/plugins - - find ../../ -maxdepth 1 -type d | \ - grep -v assemblytool | \ - while read directory; do - if [ $directory = "../../" ]; then - continue - fi - - # nuisance to get the community engine, but not the engine - if [ $(echo $directory | grep engine | wc -l) -gt 0 ]; then - if [ $(echo $directory | grep community | wc -l) -lt 1 ]; then - continue - fi - fi - - if [ ! -h $(basename $directory) ]; then - ln -s $directory - fi - done - - bash /usr/sbin/drop-and-create-ecedb \ - 1>>$log 2>>$log - cd ~/ - rm -rf /opt/escenic/engine/plugins -} - -function set_up_basic_nursery_configuration() -{ - print "Setting up the basic Nursery configuration ..." - - cp -r /opt/escenic/engine/siteconfig/config-skeleton/* \ - $common_nursery_dir/ - cp -r /opt/escenic/engine/security/ \ - $common_nursery_dir/ - - make_dir /etc/escenic/engine/instance - - for el in /opt/escenic/assemblytool/plugins/*; do - if [ ! -d $el/misc/siteconfig/ ]; then - continue - fi - - cp -r $el/misc/siteconfig/* $common_nursery_dir/ - - done - - cat > $common_nursery_dir/ServerConfig.properties < $common_nursery_dir/neo/io/managers/ContentManager.properties < $file.tmp - mv $file.tmp $file - - file=$common_nursery_dir/com/escenic/webstart/StudioConfig.properties - cat >> $file < $el/RMI.properties - fi - done + print_and_log "Installing common OS packages ..." + local packages="curl wget unzip bc ca-certificates" + local git_package=git + + if [ $on_debian_or_derivative -eq 1 ]; then + # Ubuntu doesn't have git (!) but only git-core. + if [ $on_ubuntu -eq 1 ]; then + git_package=git-core + fi + packages="${packages} bind9-host" + elif [ $on_redhat_or_derivative -eq 1 ]; then + packages="${packages} bind-utils" + fi + + install_packages_if_missing $packages $git_package + assert_commands_available lsb_release curl host wget git unzip + + # all hosts need the system-info package, hence we'll install the + # escenic-content-engine package here if on a Debian based + # system. If not, we'll just leave it, it'll be made available on + # ther systems through git. + if [ $on_debian_or_derivative -eq 1 ]; then + if [ ${fai_offline_mode-0} -eq 0 ]; then + curl -s http://apt.vizrt.com/archive.key 2>> $log | \ + apt-key add - 1>> $log 2>> $log + local package_pool=${fai_apt_vizrt_pool-unstable} + add_apt_source "deb http://apt.vizrt.com ${package_pool} main" + fi + install_packages_if_missing escenic-common-scripts + fi - nursery_context=neo/io/managers/HubConnectionManager.properties - file=/etc/escenic/engine/instance/$instance_name/$nursery_context - make_dir $(dirname $file) - - # we don't touch it if the file already exists. - if [ ! -e $file ]; then - echo "hostname=$HOSTNAME" >> $file - fi - } -function set_up_proper_logging_configuration() -{ - print "Setting up proper log4j & Java logging configuration ..." - - cat > $common_nursery_dir/trace.properties < $tomcat_base/conf/logging.properties < $instance_conf_file.tmp - mv $instance_conf_file.tmp $instance_conf_file - fi - fi - - echo "$1=$2" >> $instance_conf_file -} - -# Installs third party packages needed by the ECE (i.e. Java related). -# Also see install_common_os_packages for packages common to all -# servers in the architecture. -function install_ece_third_party_packages -{ - print "Installing 3rd party packages needed by ECE ..." - - if [ $on_debian_or_derivative -eq 1 ]; then - - if [ $on_ubuntu -eq 1 ]; then - add_apt_source "deb http://archive.canonical.com/ $(lsb_release -s -c) partner" - fi - - echo "sun-java6-jdk shared/accepted-sun-dlj-v1-1 boolean true" | \ - debconf-set-selections - - packages=" - ant - ant-contrib - ant-optional - libapr1 - libtcnative-1 - libmysql-java - memcached - maven2 - sun-java6-jdk - wget" - - install_packages_if_missing $packages - fi - - for el in ant mvn java; do - assert_pre_prequesite $el - done -} - -function set_up_app_server -{ - print "Setting up the application server ..." - - if [ $fai_enabled -eq 0 ]; then - print "On which ports do you wish to run the app server on?" - print "Press ENTER to accept the default (port 8080, shutdown port 8005)" - print "Or enter: , e.g.: '8180 8105'" - echo -n "Your choice [8080 8005]> " - read user_ports - else - user_ports=$(get_conf_value fai_editor_port) - if [ -n "${user_ports}" ]; then - user_ports=$user_ports" "$(get_conf_value fai_editor_shutdown) - fi - fi - - if [ -z "$user_ports" ]; then - appserver_port=8080 - shutdown_port=8005 - else - appserver_port=$(echo $user_ports | cut -d' ' -f1) - shutdown_port=$(echo $user_ports | cut -d' ' -f2) - fi - - debug "user_ports=[${user_ports}] appserver_port=$appserver_port shutdown_port=$shutdown_port" - - if [ $fai_enabled -eq 0 ]; then - print "Another question: Where does the database run?" - print "Press ENTER to accept the default ($HOSTNAME:3306)" - print "Or enter: :, e.g.: 'db1:3306'" - echo -n "Your choice [$HOSTNAME:3306]> " - read user_database - else - user_database=$(get_conf_value fai_db_host) - if [ -n "${user_database}" ]; then - user_database=$user_database":"$(get_conf_value fai_db_port) - fi - fi - - if [ -z "$user_database" ]; then - db_host=$HOSTNAME - db_port=3306 - else - db_host=$(echo $user_database | cut -d':' -f1) - db_port=$(echo $user_database | cut -d':' -f2) - fi - - if [ $fai_enabled -eq 0 ]; then - print "Last question: Where does the search instance run?" - print "Press ENTER to accept the default ($HOSTNAME:8080)" - print "If you're in doubt, just press ENTER :-)" - print "Or enter: :, e.g.: 'search1:8080'" - echo -n "Your choice [$HOSTNAME:8080]> " - read user_search - else - user_search=$(get_conf_value fai_search_host) - if [ -n "${user_search}" ]; then - user_search=${user_search}":"$(get_conf_value fai_search_port) - fi - fi - - if [ -z "$user_search" ]; then - search_host=$HOSTNAME - search_port=8080 + if [ -z "$instance_name" ]; then + instance_conf_file=$escenic_conf_dir/ece.conf else - search_host=$(echo $user_search | cut -d':' -f1) - search_port=$(echo $user_search | cut -d':' -f2) + instance_conf_file=$escenic_conf_dir/ece-$instance_name.conf fi - - cd $download_dir - wget --continue $tomcat_download \ - 1>>$log 2>>$log - - cd /opt/ - - tar xzf $download_dir/apache-tomcat*.tar.gz - ln -sf apache-tomcat* tomcat - - tomcat_base=/opt/tomcat-${instance_name} - make_dir $tomcat_base - - cp -r /opt/apache-tomcat*/conf/ $tomcat_base - for el in bin escenic/lib lib work logs temp webapps; do - make_dir $tomcat_base/$el - done - - set_ece_instance_conf tomcat_base $tomcat_base - set_ece_instance_conf tomcat_home /opt/tomcat - - cd $tomcat_base/lib - make_ln $jdbc_driver - - cat $tomcat_base/conf/catalina.properties | \ - sed 's/common.loader=/common.loader=\$\{catalina.base\}\/escenic\/lib\/\*\.jar\,/g' \ - > $tomcat_base/conf/catalina.properties.tmp - mv $tomcat_base/conf/catalina.properties.tmp $tomcat_base/conf/catalina.properties - - cat > $tomcat_base/conf/server.xml < - - - - - - - - - - - - - - - - - - - - - - - + set_conf_file_value $1 $2 $instance_conf_file +} +# $1 the domain +# +# The method will ensure that the passed domain is resolvable by the +# host on which ece-install is run. +function ensure_domain_is_known_to_local_host() { + if [ -z "$1" ]; then + return 1 + fi + + local hostname_ip=$(ping -c 1 $HOSTNAME 2>/dev/null | \ + head -1 | \ + cut -d'(' -f2 | \ + cut -d')' -f1) + local domain_ip=$(ping -c 1 $1 2>/dev/null | \ + head -1 | \ + cut -d'(' -f2 | \ + cut -d')' -f1) + + local keep_off_etc_hosts=${fai_keep_off_etc_hosts-0} + if [[ $domain_ip != "127.0.0.1" && \ + $domain_ip != "127.0.1.1" && \ + $domain_ip != $hostname_ip && \ + $keep_off_etc_hosts -ne 1 ]]; then + print_and_log "The domain name ${1} is not resolvable to this host" + print_and_log "I will remedy this by adding it to /etc/hosts" + cat >> /etc/hosts < $tomcat_base/conf/context.xml < - - - WEB-INF/web.xml - - - - - - - - - - - - - - - - -EOF - + fi + if [[ $domain_ip != $localhost_ip && \ + $domain_ip != hostname_ip && \ + $keep_off_etc_hosts -eq 1 ]]; then + print_and_log "The domain name ${1} is not resolvable to this host" + print_and_log "but I will keep off /etc/hosts as you've requested." + fi } -# last, give the control back to the ECE user & group +# last, give the control back to the ECE user & group function set_correct_permissions() { - print "Setting correct permissions on all ECE related directories ..." - - if [ $(grep $ece_user /etc/passwd | wc -l) -lt 1 ]; then - # TODO add support for useradd - adduser $ece_user - fi - if [ $(grep $ece_group /etc/group | wc -l) -lt 1 ]; then - addgroup $ece_group - fi - - for el in $dir_list; do - if [ -d $el ]; then - chown -R ${ece_user}:${ece_group} $el \ - 1>>$log 2>>$log - fi - done - - if [ -d "$tomcat_base" ]; then - chown -R ${ece_user}:${ece_group} $tomcat_base \ - 1>>$log 2>>$log - fi - - if [ $(ls /tmp | grep ^ece | wc -l) -gt 0 ]; then - chown -R ${ece_user}:${ece_group} /tmp/ece-* \ - 1>>$log 2>>$log - fi + log "Setting correct permissions on ECE related directories ..." + + for el in $engine_dir_list; do + if [ ! -d $el ]; then + continue + fi + + if [ ${el} = ${escenic_data_dir} ]; then + local correct_permission=$(find ${el} \ + -maxdepth 0 \ + -user ${ece_user} | \ + wc -l) + if [ $correct_permission -gt 0 ]; then + log "Data directory root, $el," + log "has correct permissions, skiping sub directories." + continue + fi + fi + + run chown -R ${ece_user}:${ece_group} $el + done + + if [ -d "$tomcat_base" ]; then + run chown -R ${ece_user}:${ece_group} $tomcat_base + fi + + # it's important that the ECE user has access to read the + # tomcat_home files, e.g. when creating a backup of it. + if [ -d "$tomcat_home" ]; then + run chgrp -R ${ece_group} $tomcat_home + run chmod -R g+rx $tomcat_home + if [ -h "$tomcat_home" ]; then + local file=$(dirname $tomcat_home)/$(readlink $tomcat_home) + run chgrp -R ${ece_group} ${file} + run chmod -R g+rx $tomcat_home ${file} + fi + fi } - function print_status_and_next_steps() { - # su - $ece_user -c "ece versions" 1>>$log 2>>$log - - now=`date +%s` - started=`stat -c %Y $pid_file` - seconds=$(( now - started )) - days=$(( seconds / ( 60 * 60 * 24 ) )) - seconds_left=$(( seconds - ( $days * 60 * 60 * 24 ) )) - hours=$(( seconds_left / ( 60 * 60 ) )) - seconds_left=$(( seconds_left - ( $hours * 60 * 60 ) )) - minutes=$(( seconds_left / 60 )) - seconds_left=$(( seconds_left - $minutes * 60 )) - - print "The basic setup of $instance_name is now completed!" - print "It took" ${days}d ${hours}h ${minutes}m ${seconds_left}s - - if [ $install_profile_number -eq $PROFILE_CACHE_SERVER ]; then - echo "The cache server up and running at http://${HOSTNAME}:80/" - fi - - if [ $install_profile_number -eq $PROFILE_EDITORIAL_SERVER -o \ - $install_profile_number -eq $PROFILE_PRESENTATION_SERVER ]; then - print "Your app server is running on"\ "http://${HOSTNAME}:$appserver_port" - print "" - versions="ece -i $instance_name -t $type versions" - print "You can now switch to the $ece_user and type:" - print " $versions" - print "to see the Escenic software I have installed for you." - - print "" - print "You're next steps on ${HOSTNAME} will be:" - print "- configure the instances you want to be started at boot time" - print " in /etc/default/ece" - print "- include $instance_name in your RMI hub configuration." - print "" - print "Enjoy your time with Escenic Content Engine!" - print "" - print "-Vizrt Online" - fi -} - -# Based on Erik Mogensen's work: -# //depot/branches/personal/mogsie/fromscratch/create-publication.sh -function create_publication_in_db() -{ - print "Creating publication ${publication_name} using $instance_name ..." - - ece_admin_uri=http://$HOSTNAME:${appserver_port}/escenic-admin - - cookie=$(HEAD $ece_admin_uri | \ - grep -i "^Set-Cookie" | \ - sed s/.*'JSESSIONID=\([^;]*\).*'/'\1'/) - - if [ "$cookie" == "" ] ; then - print "Unable to get a session cookie." - exit 1; - fi - - curl --silent \ - -F "type=webapp" \ - -F "resourceFile=@${1}" \ - --cookie JSESSIONID="$cookie" \ - "${ece_admin_uri}/do/publication/resource" \ - 1>>$log 2>>$log - - curl --silent \ - -F "name=${publication_name}" \ - -F "publisherName=Escenic" \ - -F "adminPassword=${publication_name}" \ - -F "adminPasswordConfirm=${publication_name}" \ - --cookie JSESSIONID="$cookie" \ - "${ece_admin_uri}/do/publication/insert" \ - 1>>$log 2>>$log -} - -function create_publication_definition_and_war() -{ - publication_name=mypub - - if [ $fai_enabled -eq 0 ]; then - print "What name do you wish to give your publication?" - print "Press ENTER to accept ${publication_name}" - echo -n "Your choice [${publication_name}]> " - read publication_name + local now=`date +%s` + local started=`stat -c %Y $pid_file` + local seconds=$(( now - started )) + local days=$(( seconds / ( 60 * 60 * 24 ) )) + local seconds_left=$(( seconds - ( $days * 60 * 60 * 24 ) )) + local hours=$(( seconds_left / ( 60 * 60 ) )) + local seconds_left=$(( seconds_left - ( $hours * 60 * 60 ) )) + local minutes=$(( seconds_left / 60 )) + local seconds_left=$(( seconds_left - $minutes * 60 )) + + if [ $install_profile_number -ne $PROFILE_RESTORE_FROM_BACKUP ]; then + local message="The installation is now $(green complete)!" else - publication_name=$(get_conf_value fai_publication_name) + local message="The restore is now $(green complete)!" fi + print_and_log "${message} It took" ${days}d ${hours}h ${minutes}m ${seconds_left}s - if [ -z "$publication_name" ]; then - publication_name=mypub + if [ $install_profile_number -ne $PROFILE_RESTORE_FROM_BACKUP -a \ + $install_profile_number -ne $PROFILE_CACHE_SERVER -a \ + $install_profile_number -ne $PROFILE_WIDGET_FRAMEWORK ]; then + add_next_step "Install info: "\ "/usr/share/doc/escenic/ece-install-guide.txt" + add_next_step "Guide books: http://documentation.vizrt.com/ece-5.5.html" fi - print "Setting up the ${publication_name} publication ..." - make_dir /opt/escenic/assemblytool/publications/ - cd /opt/escenic/assemblytool/publications/ - cat > /opt/escenic/assemblytool/publications/${publication_name}.properties <>$log 2>>$log - cp target/demo-core-*.war ${publication_war} - else - print "Basing your ${publication_name}.war on ECE/demo-clean ..." - cp /opt/escenic/engine/contrib/wars/demo-clean.war ${publication_war} - fi - + print $'\n'"Enjoy your time with Escenic Content Engine!"$'\n' + print "-$(red Vizrt) Online" } function check_for_required_downloads() { - if [ $ece_software_setup_completed -eq 1 ]; then - return - fi - - print "Asserting that required downloads succeeded ..." - required_escenic_packages="engine assemblytool" - - for el in $required_escenic_packages; do - if [ $(ls $download_dir | \ - grep ${el} | \ - grep -v community | \ - grep -v analysis | \ - grep .zip$ | \ - wc -l) \ - -lt 1 ]; then - print "Couldn't find $el* in $download_dir" - exit 1 - fi - done -} - -function install_memcached() -{ - if [ $on_debian_or_derivative ]; then - install_packages_if_missing "memcached" - fi - - # 1) download java library - # 2) configure PresenationArticle cache for the publication - # 3) build and re-deploy - - cat >> /etc/escenic/engine/common/Initial.properties <> /home/$ece_user/.bashrc - fi - if [ $(grep JAVA_HOME /root/.bashrc | wc -l) -lt 1 ]; then - echo JAVA_HOME=$java_home >> /root/.bashrc - fi - - if [ $on_debian_or_derivative -eq 1 ]; then - export JAVA_HOME=$java_home - fi - - if [ $(grep bash_completion.d/ece /home/$ece_user/.bashrc | wc -l) -lt 1 ] - then - echo ". /etc/bash_completion.d/ece" \ - >> /home/$ece_user/.bashrc - fi -} - -function set_up_solr() -{ - print "Setting up solr ..." - if [ ! -d /etc/escenic/solr ]; then - cp -r /opt/escenic/engine/solr/conf /etc/escenic/solr - fi - - make_dir /var/lib/escenic/solr/ - cd /var/lib/escenic/solr/ - if [ ! -h conf ]; then - ln -s /etc/escenic/solr conf - fi -} - -# So far, I've used this method for copy/past-ing it into the shell -# before running ece-insatll anew. It might be useful for its own -# command later, though. -function un_install_ece() -{ - print "Uninstalling ECE ..." - # TODO safety, warnings++ - rm -rf /etc/escenic/ \ - /opt/*tomcat* \ - /opt/escenic \ - /var/lib/escenic \ - /var/run/escenic/ \ - /etc/escenic/ \ - /usr/bin/ece \ - /var/log/escenic/ \ - /etc/apt/sources.list.d/escenic.list \ -# $HOME/.m2 \ - /var/run/ece-install.pid - - apt-get --yes --purge remove \ - varnish \ - percona* \ - maven2 \ - ant \ - ant-contrib \ - ant-optional \ - libmysql-java \ - memcached \ - sun-java6-jdk - - apt-get clean - - # prepare for new install - mv /tmp/*.zip /tmp/ece-downloads/ -} - -function stop_conflicting_processes() -{ - print "Stopping conflicting processes ..." - # TODO this is dirty - killall java 1>>$log 2>>$log + if [ $ece_software_setup_completed -eq 1 ]; then + return + fi + + print_and_log "Asserting that required downloads succeeded ..." + local required_escenic_packages="engine" + # TODO: make assembly tool required if not using an EAR. + if [ $install_profile_number -eq $PROFILE_ANALYSIS_SERVER ]; then + required_escenic_packages="analysis-engine" + fi + + local some_is_missing=0 + + for el in $required_escenic_packages; do + if [ $(ls $download_dir/$el*.{zip,jar} 2>/dev/null | wc -l) -lt 1 ]; then + print_and_log "-> $el is missing" + some_is_missing=1 + else + # want the newest one if there are several + local file=$(ls $download_dir/$el*.{zip,jar} 2>/dev/null | tail -1) + unzip -t $file > /dev/null 2>&1 + + if [ $? -ne 0 ]; then + log "$file has been downloaded, but is faulty," + log "remove it and re-run $0" + return 9 + fi + fi + done + + if [ $some_is_missing -eq 1 ]; then + print_and_log "Add these download archives to the technet_download_list" + print_and_log "and re-run $(basename $0)" + exit 1 + fi } -function set_up_varnish() +function set_up_user_environment() { - print "Setting up Varnish to match your environment ..." - /etc/init.d/varnish stop 1>>$log 2>>$log - - file=/etc/default/varnish - cat $file | \ - sed 's/6081/80/g' \ - > $file.tmp - mv $file.tmp $file - - cat > /etc/varnish/default.vcl <> /etc/varnish/default.vcl <> /etc/varnish/default.vcl <> /etc/varnish/default.vcl <> /etc/varnish/default.vcl <> /etc/varnish/default.vcl <> /etc/varnish/default.vcl <> /etc/varnish/default.vcl < 0) { - set resp.http.X-Cache = "HIT #" + obj.hits; - } - else { - set resp.http.X-Cache = "MISS"; - } - - set resp.http.X-Cache-Backend = req.backend; -} + if [ $(grep bash_completion.d/ece ${bashrc} 2>/dev/null | \ + wc -l) -lt 1 ]; then + cat >> ${bashrc} <>$log 2>>$log + fi + + # Having the EDITOR variable set is requirement for several + # commmands, including running: + # + # 'ece -i -p -r edit' + if [ $(grep EDITOR ${bashrc} 2>/dev/null | wc -l) -lt 1 ]; then + echo "export EDITOR=vi" >> ${bashrc} + fi + + if [ $(grep ECE_CONF_LOCATIONS ${bashrc} | wc -l) -eq 0 ]; then + echo "export ECE_CONF_LOCATIONS=\"$escenic_conf_dir\"" >> ${bashrc} + run chown ${ece_user}:${ece_group} ${bashrc} + fi } function read_user_input() { - echo "Hi, which server profile do you wish to install?" - echo "" - echo "Select 1-7 and press ENTER" - echo "" - echo " $PROFILE_ALL_IN_ONE - All in one, the full stack on one host. " - echo " Suitable for development and test environments." - echo " It will install caching server, ECE, assembly host, database &" - echo " Widget Framework as well as set up a new publication." - echo " $PROFILE_EDITORIAL_SERVER - Editorial (publication) server." - echo " This will install ECE, an assembly host & as well as " - echo " creating and set up a new publication." - echo -n " $PROFILE_PRESENTATION_SERVER - Presentation server " - echo "(ECE, memcached)." - echo " $PROFILE_DB_SERVER - Database server." - echo -n " $PROFILE_CACHE_SERVER - Cache server " - echo "(cache & web server)." - echo " $PROFILE_RMI_HUB - RMI hub." - echo -n " $PROFILE_SEARCH_SERVER - Standalone search instance " - echo "(solr + indexer-webapp)." - echo " $PROFILE_WIDGET_FRAMEWORK - Install the Widget Framework." - echo " (You need user/pass for repo.escenic.com for this)." - echo " $PROFILE_CREATE_PUBLICATION - Create a new publication." - echo " This profile will create new publication based on WF" - echo " (if available) or ECE/clean-demo." - echo "" + installation_profiles=( + "$PROFILE_ALL_IN_ONE - All in one, full stack on one host,"\ +" suitable for dev & test environments" +"$PROFILE_EDITORIAL_SERVER - Editorial (publication) server" +"$PROFILE_PRESENTATION_SERVER - Presentation server (ECE + memcached)." +"$PROFILE_DB_SERVER - Database server" +"$PROFILE_CACHE_SERVER - Cache server (cache and web server)" +"$PROFILE_RMI_HUB - RMI hub" +"$PROFILE_SEARCH_SERVER - Search server (Solr + indexer-webapp)" +"$PROFILE_WIDGET_FRAMEWORK - Install Widget Framework." +"$PROFILE_RESTORE_FROM_BACKUP - Restore from backup"\ +" (DB, data files, binaries, conf & publications)" + ) + + echo "Hi, which server profile do you wish to install?"$'\n' + + for (( i = 0; i < ${#installation_profiles[@]}; i++ )); do + echo " " ${installation_profiles[$i]} + done + + echo $'\n'"Select 1-${#installation_profiles[@]} and press ENTER" echo -n "Your choice [1]> " read install_profile_number - + if [ -z "$install_profile_number" ]; then install_profile_number=$PROFILE_ALL_IN_ONE fi - + + if [ $(is_number $install_profile_number) -eq 0 ]; then + print_and_log "Profile number, $install_profile_number, is not a number" + remove_pid_and_exit_in_error + fi } function assert_correct_runtime_environment() { if [ $(whoami) != "root" ]; then - print "You must be root when running $(basename $0)" + echo "You must be root when running $(basename $0)" exit 1 fi - - if [ -e $pid_file ]; then - print "There's already one $(basename $0) process running. If you believe" - print "this is wrong, e.g. if a previous run of $(basename $0) was aborted" - print "before it completed, you may delete ${pid_file} and " - print "run $(basename $0) again." - exit 1 + + create_pid + started=`stat -c %Y $pid_file` + create_lock + + if [ ! -e "$conf_file" ]; then + print_and_log $conf_file "doesn't exist." \ + "I cannot live without it, so I'm exiting :-(" + remove_pid_and_exit_in_error else - echo $BASHPID > $pid_file + source $conf_file fi } -function common_pre_install() -{ - print "I'm logging to $log" +## checks the Technet and WF credentials. +function check_software_credentials() { + install_packages_if_missing curl + assert_commands_available curl + + if [ ${fai_offline_mode-0} -eq 1 ]; then + return + fi + + if [ ${fai_skip_password_checks-0} -eq 1 ]; then + return + fi + + print_and_log "Verifying that Technet credentials are OK ..." + + local url=http://technet.escenic.com + local notok=$( + is_unauthorized_to_access_url \ + ${technet_user} \ + ${technet_password} \ + $url + ) + if [ $notok -eq 1 ]; then + print_and_log "The technet_user & technet_password in $conf_file" + print_and_log "are insufficient to access ${url} :-(" + remove_pid_and_exit_in_error + fi + + # this method is called so early, so we don't yet know which + # installation profile is being used. Hence, we just test for if the + # wf_user & password has been set and are able to access the Escenic + # Maven repository. + + # if the WF credentials are set, we check to see if they're OK + if [ -n "${wf_user}" -a -n "${wf_password}" ]; then + print_and_log "Verifying that Widget Framework credentials are OK ..." + local url=http://maven.vizrt.com + local notok=$( + is_unauthorized_to_access_url \ + ${wf_user} \ + ${wf_password} \ + $url + ) + if [ $notok -eq 1 ]; then + print_and_log "The wf_user & wf_password in $conf_file" + print_and_log "are insufficient to access ${url} :-(" + remove_pid_and_exit_in_error + fi + elif [ ${fai_enabled-0} -eq 1 ]; then + if [[ ${fai_wf_install-0} -eq 1 || ${fai_all_install-0} -eq 1 ]]; then + print_and_log "You must have a valid wf_user & wf_password" + print_and_log "in your ${conf_file}. If you don't have these," + print_and_log "please contact support@escenic.com" + remove_pid_and_exit_in_error + fi + fi +} - technet_user=`grep technet_user $conf_file 2>/dev/null | cut -d'=' -f2` - technet_password=`grep technet_password $conf_file 2>/dev/null | cut -d'=' -f2` +function common_pre_install() { + print "I'm logging to $log" - if [ -z "$technet_user" -o -z "$technet_password" ]; then - print "Be sure to set technet_user and technet_password " - print "in $conf_file" - exit 1 - fi + run source $conf_file + + # These variables are placed here as all the directories can be + # overridden in ece-install.conf + common_nursery_dir=$escenic_conf_dir/engine/common + + # Because of issue VF-3559, we also create the default family and + # host directories (fixed in 5.4.0.x). + engine_dir_list=" + $common_nursery_dir + $escenic_conf_dir/engine/family/default + $escenic_conf_dir/engine/environment + $escenic_conf_dir/engine/host/localhost + $escenic_conf_dir/engine/instance + $escenic_root_dir + $escenic_cache_dir + $escenic_crash_dir + $escenic_data_dir + $escenic_data_dir/solr/data + $escenic_log_dir + $escenic_run_dir + $escenic_spool_dir/import + " + ece_install_env=${escenic_run_dir}/$(basename $0).env + ensure_variable_is_set technet_user technet_password + wget_auth="--http-user ${technet_user} --http-password ${technet_password}" if [ -e /etc/debian_version -a -x /usr/bin/dpkg ]; then on_debian_or_derivative=1 export DEBIAN_FRONTEND=noninteractive - fi - - if [ $(lsb_release -i | grep Ubuntu | wc -l) -gt 0 ]; then - on_ubuntu=1 - elif [ $(lsb_release -i | grep Debian | wc -l) -gt 0 ]; then - on_debian=1 - fi - - make_dir $download_dir - set_correct_permissions - stop_conflicting_processes - install_common_os_packages - set_up_ece_scripts - set_up_user_enviornment -} -function assert_pre_prequesite() -{ - if [ $(which $1 | wc -l) -lt 1 ]; then - print "Please install $1 and then run $(basename $0) again." - exit 1 - fi -} - -function add_apt_source() -{ - escenic_sources=/etc/apt/sources.list.d/escenic.list - if [ ! -e $escenic_sources ]; then - echo "$@" >> $escenic_sources - apt-get update 1>>$log 2>>$log - elif [ $(grep "$@" $escenic_sources | wc -l) -lt 1 ]; then - echo "$@" >> $escenic_sources - apt-get update 1>>$log 2>>$log + # chicken and the egg problem, we need lsb_release to install the + # packages later on, hence as soon as we know we've got a Debian + # based platform, we install lsb-release. Also note, the + # executable, lsb_release, is in the list of required binaries in + # install_common_os_packages. + install_packages_if_missing "lsb-release" fi -} -function install_cache_server() -{ - print "Installing a caching server on $HOSTNAME ..." + if [ -e /etc/redhat-release ]; then + on_redhat_or_derivative=1 + install_packages_if_missing "redhat-lsb" - if [ $on_debian_or_derivative -eq 1 ]; then - curl http://repo.varnish-cache.org/debian/GPG-key.txt 2>>$log | \ - apt-key add - 1>>$log 2>>$log - fi - - if [ $on_debian -eq 1 ]; then - add_apt_source "deb http://repo.varnish-cache.org/debian/ $(lsb_release -s -c) varnish-3.0" - elif [ $on_ubuntu -eq 1 ]; then - add_apt_source "deb http://repo.varnish-cache.org/ubuntu/ $(lsb_release -s -c) varnish-3.0" + if [ $(lsb_release -i | grep RedHat | wc -l) -gt 0 ]; then + on_redhat=1 + fi fi - if [ $on_debian_or_derivative -eq 1 ]; then - apt-get -y install varnish 1>>$log 2>>$log - fi - - assert_pre_prequesite varnishd - - if [ $fai_enabled -eq 0 ]; then - print "You must now list your backend servers." - print "Seperate the entries with a space. e.g.: app1:8080 app2:8080." - print "Press ENTER to accept the default: ${HOSTNAME}:${appserver_port}" - echo -n "Your choice [${HOSTNAME}:${appserver_port}]> " - read backend_servers - else - backend_servers=$(get_conf_value fai_cache_backends) - fi + check_software_credentials + assert_commands_available lsb_release - if [ -z "$backend_servers" ]; then - backend_servers="${HOSTNAME}:${appserver_port}" + if [ $(lsb_release -i | grep Ubuntu | wc -l) -gt 0 ]; then + on_ubuntu=1 + elif [ $(lsb_release -i | grep Debian | wc -l) -gt 0 ]; then + on_debian=1 fi - set_up_varnish $backend_servers -} + # git will fail if curl doesn't have the right CA certificates + # installed. As this happens on RedHat/CentOS 5.7, we turn it off + # here. + export GIT_SSL_NO_VERIFY=true -function install_database_server() -{ - print "Installing the database server on $HOSTNAME ..." - - if [ $on_debian_or_derivative -eq 1 ]; then - - code_name=$(lsb_release -s -c) - - supported_code_name=0 - supported_list="lenny squeeze hardy lucid maverick" - for el in $supported_list; do - if [ $code_name = $el ]; then - supported_code_name=1 - fi - done - - # some how, this is to install Percona 5.5 - if [ -e /var/lib/mysql/debian-*.flag ]; then - rm /var/lib/mysql/debian-*.flag - fi - - if [ $supported_code_name -eq 1 ]; then - print "Installing the Percona database ..." - - if [ $(apt-key list| grep CD2EFD2A | wc -l) -lt 1 ]; then - gpg --keyserver hkp://keys.gnupg.net \ - --recv-keys 1C4CBDCDCD2EFD2A \ - 1>>$log 2>>$log - gpg -a --export CD2EFD2A | apt-key add - \ - 1>>$log 2>>$log - fi - add_apt_source "deb http://repo.percona.com/apt ${code_name} main" - packages="percona-server-server percona-server-client" - install_packages_if_missing $packages - else - print -n "The Percona APT repsository " - print "doesn't have packages for your Debian (or derivative) " - print "version with code name $code_name. I will use vanilla " - print "MySQL instead." + make_dir $download_dir + install_common_os_packages + create_user_and_group_if_not_present $ece_user $ece_group + set_up_user_environment - packages="mysql-server mysql-client libmysql-java" - install_packages_if_missing $packages - fi + if [ ${keep_off_wget_user_agent-0} -eq 0 ]; then + wget_opts="${wget_opts} + --header User-Agent:wget/$(basename $0)-${ece_scripts_version} + " fi - - assert_pre_prequesite mysql - - download_escenic_components - set_up_engine_and_plugins - set_up_ecedb } # $1 is the default instance name, the calee is responsible for # setting this. function ask_for_instance_name() { - if [ $fai_enabled -eq 0 ]; then + if [ ${fai_enabled-0} -eq 0 ]; then print "What do you want to call this ECE instance?" print "Press ENTER to accept the default instance name, $1." echo -n "Your choice [$1]> " read instance_name + else + if [ $install_profile_number -eq $PROFILE_EDITORIAL_SERVER ]; then + instance_name=${fai_editor_name-$1} + elif [ $install_profile_number -eq $PROFILE_PRESENTATION_SERVER ]; then + instance_name=${fai_presentation_name-$1} + elif [ $install_profile_number -eq $PROFILE_SEARCH_SERVER ]; then + instance_name=${fai_search_name-$1} + elif [ $install_profile_number -eq $PROFILE_ALL_IN_ONE ]; then + instance_name=${fai_all_name-$1} + elif [ $install_profile_number -eq $PROFILE_ANALYSIS_SERVER ]; then + instance_name=${fai_analysis_name-$1} + fi fi if [ -z "$instance_name" ]; then instance_name=$1 fi - make_dir /etc/escenic/engine/instance/${instance_name} + make_dir $escenic_conf_dir/engine/instance/${instance_name} } -# $1= -# $2= -function install_ece_instance() -{ - install_ece_third_party_packages - - ask_for_instance_name $1 - download_escenic_components - check_for_required_downloads - - set_up_engine_and_plugins - set_up_assembly_tool - set_up_basic_nursery_configuration - set_up_instance_specific_nursery_configuration - - set_up_app_server - set_up_proper_logging_configuration - - # special treatment for presentation servers - if [ $2 -eq $PROFILE_PRESENTATION_SERVER ]; then - file=/etc/escenic/ece-${instance_name}.conf - print "Creating instance specific conf: $file ..." - cat >> $file < /etc/default/ece <>$log 2>>$log + print_and_log "Installing a presentation server on $HOSTNAME." + type=engine + install_ece_instance $default_ece_intance_name $PROFILE_PRESENTATION_SERVER } -function ensure_that_instance_is_running() +function install_editorial_server() { - ece_command="ece -i $1 -t $type status" - if [ $(su - $ece_user -c "$ece_command" | grep UP | wc -l) -lt 1 ]; then - ece_command="ece -i $1 -t $type start" - su - $ece_user -c "$ece_command" 1>>$log 2>>$log - # TODO improve this by adding a timed while loop - sleep 60 - fi + print_and_log "Installing an editorial server on $HOSTNAME ..." + type=engine + install_ece_instance "editor1" 0 } -function create_publication() +function run_hook() { - if [ ! -e /opt/escenic/engine -o \ - ! -e /opt/escenic/assemblytool ]; then - print "Please install ECE and an assembly environment before" - print "running this installation profile again." - exit 1 - fi - - print "Getting ready to create a new publiation ..." - create_publication_definition_and_war + if [ -e $ece_install_scripts_dir -a \ + -e ${ece_install_scripts_dir}/${1} ]; then - # TODO make educated guesses about the available instances on - # $HOSTNAME by looking in /etc/escenic/ece-*.conf - - default_instance=dev1 + # Dumping all set variables (no functions) to a file from + # which the hooks can pick them up. We do this to avoid + # running "export" in front of all local variables which may + # or may not be useful. Furthermore, we filter out upper case + # variables as these are environment variables. + set | grep ^[a-z] | grep \= > ${ece_install_env} - if [ $fai_enabled -eq 0 ]; then - print "Which ECE instance do you wish to use to create it?" - echo -n "Your choice [$default_instance]> " - read instance_name - else - instance_name=$(get_conf_value fai_publication_use_instance) - fi - - if [ -z "$instance_name" ]; then - instance_name=$default_instance + print_and_log "Started hook $1 ..." + bash ${ece_install_scripts_dir}/${1} + print_and_log "Finished hook $1 ..." fi - - type=engine - ensure_that_instance_is_running $instance_name - # TODO set the appserver_port based on the instance_name chosen - # above. - create_publication_in_db $publication_war - assemble_deploy_and_restart_type } -function install_editorial_server() +# useful for development and test environments. +function install_all_in_one_environment() { - print "Installing an editorial server on $HOSTNAME ..." - - install_ece_instance "editor1" 0 + print_and_log "Installing an all-in-one environment on $HOSTNAME ..." type=engine - ece_command="ece -i $instance_name -t $type assemble deploy restart" - su - $ece_user -c "$ece_command" 1>>$log 2>>$log + install_database_server + install_ece_instance $default_ece_intance_name 0 + install_cache_server + install_web_server 2 + set_up_solr + + install_widget_framework create_publication } -function install_rmi_hub() -{ - make_dir /etc/escenic/rmi-hub - - cp -r /opt/escenic/engine/contrib/rmi-hub/config/* \ - /etc/escenic/rmi-hub/ +## We're adding the ece script to the run levels regardless of the +## installation profile (as long as the init.d script exists) since +## it's possible to install several profiles from the same run of +## ece-install and then it's not sure the test for a profile that +## needs the ece init.d script will succeed. +function add_ece_init_to_runlevels() { + if [ ! -x /etc/init.d/ece ]; then + return + fi + + if [ $on_debian_or_derivative -eq 1 ]; then + print_and_log "Adding the ece init.d script to the default run levels ..." + run update-rc.d ece defaults 35 + elif [ $on_redhat_or_derivative -eq 1 ]; then + run chkconfig --level 35 ece on + else + add_next_step "Remember to add /etc/init.d/ece to the desired run levels." + # TODO add init.d to the default runlevels, for other + # distributions too: + # - Gentoo: rc-update add ece default + fi +} - hub_host=$HOSTNAME - file=$common_nursery_dir - file=$file/neo/io/managers/HubConnectionManager.properties +function common_post_install() { + add_ece_init_to_runlevels - make_dir $(basename $file) - echo "hub=rmi://${hub_host}:1099/hub/Hub" > $file - cat > common/io/api/EventManager.properties <> $ece_install_trail_file } -function install_widget_framework() -{ - # TODO java.lang.NoClassDefFoundError: - # Lcom/escenic/framework/captcha/ReCaptchaConfig; - - wf_user=$(get_conf_value wf_user) - wf_password=$(get_conf_value wf_password) - - if [ -z "$wf_user" -o -z "$wf_password" ]; then - print "Be sure to set wf_user and wf_password in $conf_file" - print "If you don't have these, please contact support@escenic.com" - exit 1 - fi - - print "Creating a Maven settings file: $HOME/.m2/settings.xml ..." - # TODO consider //${ece_user} instead - make_dir $HOME/.m2 - cat > $HOME/.m2/settings.xml < - - - - escenic-repo - ${wf_user} - ${wf_password} - - - - - - escenic-profile - - - escenic-repo - Repository for EWF libraries - http://repo.escenic.com/ - default - - - - - - - escenic-profile - - -EOF - - print "Downloading Widget Framework from technet.escenic.com ..." - for el in $wf_download_list; do - cd $download_dir - wget --continue \ - --http-user $technet_user \ - --http-password $technet_password \ - $el \ - 1>>$log 2>>$log - - cd /opt/escenic/ - unzip -u $download_dir/$(basename $el) \ - 1>>$log 2>>$log - done - - assert_pre_prequesite mvn - wf_maven_dir=$(echo /opt/escenic/widget-framework-core-*/maven) - cd $wf_maven_dir - print "Installing Widget Framework into your local Maven repository ..." - export JAVA_HOME=$java_home - mvn install \ - 1>>$log 2>>$log - - # installing the widget-framework-common as a ECE plugin - wf_dist_dir=$(echo $wf_maven_dir/widget-framework-common/target/widget-framework-common-*-dist/widget-framework-common-*) - cd /opt/escenic/assemblytool/plugins - if [ ! -h $(basename $wf_dist_dir) ]; then - ln -s $wf_dist_dir - fi - - cp -r $wf_dist_dir/misc/siteconfig/* $common_nursery_dir/ +function read_user_input() { + for el in "$@"; do + if [ $el = "-v" -o $el = "--verbose" ]; then + debug=1 + elif [ $el = "-V" -o $el = "--version" ]; then + echo "Version:" $ece_scripts_version + exit 0 + elif [ $el = "-f" -o $el = "--conf-file" ]; then + next_is_conf_file=1 + elif [[ -n $next_is_conf_file && $next_is_conf_file -eq 1 ]]; then + conf_file=$el + case ${conf_file} in + /*) + ;; + *) + conf_file=$(pwd)/${conf_file} + ;; + esac + + next_is_conf_file=0 + fi + done } -function install_search_instance() -{ - type=search1 - install_ece_instance "search1" 0 - - file=/etc/escenic/ece-${instance_name}.conf - print "Creating instance specific conf: $file ..." - cat >> $file <> $dir/SolrSearchEngine.properties - - dir=$common_nursery_dir/com/escenic/webservice/search - make_dir $dir - echo "solrURI=http://${search_host}:${search_port}/solr/select" \ - >> $dir/DelegatingSearchEngine.properties - - dir=$common_nursery_dir/com/escenic/lucy - echo "solrURI=http://${search_host}:${search_port}/solr" \ - >> $dir/LucySearchEngine.properties - - dir=$common_nursery_dir/com/escenic/forum/search/lucy - echo "solrURI=http://${search_host}:${search_port}/solr" \ - >> $dir/SearchEngine.properties - - set_up_solr - assemble_deploy_and_restart_type +function get_java_package_name() { + # The script will install oracle java installer package on Debian based + # systems and this is the path of the JAVA home with this package. If + # you're using a different system or have other preferences, change + # java_home in your ece-install.conf + default_java_version=1.6 + server_java_version=${fai_server_java_version-${default_java_version}} + if [ $server_java_version = "1.6" ]; then + java_package_name="oracle-java6-installer" + elif [ $server_java_version = "1.7" ]; then + java_package_name="oracle-java7-installer" + #elif [ $server_java_version = "1.8" ]; then + # java_package_name="oracle-java8-installer" + else + # Setting it back to default + java_package_name="" + fi } -# useful for development and test environments. -function install_all_in_one_environment() -{ - print "Installing an all-in-one environment on $HOSTNAME ..." +read_user_input "$@" +init +assert_correct_runtime_environment +get_java_package_name - install_database_server - install_ece_instance "dev1" 0 - install_cache_server - set_up_solr +if [ ${fai_enabled-0} -eq 1 ]; then + print_and_log "Full Automatic Install (FAI) enabled." + print_and_log "All user input will be read from $conf_file" - install_widget_framework - create_publication -} + common_pre_install + perform_dry_run_if_applicable -function common_post_install() -{ - print_status_and_next_steps - rm $pid_file -} + no_fai_profile=1 -for el in $@; do - if [ $el = "-v" -o $el = "--verbose" ]; then - debug=1 + if [ ${fai_un_install_everything-0} -eq 1 ]; then + install_profile_number=$PROFILE_UN_INSTALL + un_install_everything + no_fai_profile=0 fi -done - -assert_correct_runtime_environment -fai_enabled=$(get_boolean_conf_value fai_enabled) + # some profiles, like DB slave might need the NFS client, hence we + # install it pretty far up. + if [ ${fai_nfs_client_install-0} -eq 1 ]; then + install_profile_number=$PROFILE_NFS_CLIENT + install_nfs_client + no_fai_profile=0 + fi -if [ $fai_enabled -eq 1 ]; then - print "Full Automatic Install (FAI) enabled." - print "All user input will be read from $conf_file" - - if [ $(get_boolean_conf_value fai_all_install) -eq 1 ]; then + if [ ${fai_all_install-0} -eq 1 ]; then install_profile_number=$PROFILE_ALL_IN_ONE - common_pre_install install_all_in_one_environment - common_post_install - elif [ $(get_boolean_conf_value fai_editor_install) -eq 1 ]; then - install_profile_number=$PROFILE_EDITORIAL_SERVER - common_pre_install - install_editorial_server - common_post_install - elif [ $(get_boolean_conf_value fai_db_install) -eq 1 ]; then + no_fai_profile=0 + fi + + if [ ${fai_db_install-0} -eq 1 ]; then install_profile_number=$PROFILE_DB_SERVER - common_pre_install install_database_server - common_post_install - elif [ $(get_boolean_conf_value fai_cache_install) -eq 1 ]; then - install_profile_number=$PROFILE_CACHE_SERVER - common_pre_install - install_cache_server - common_post_install - elif [ $(get_boolean_conf_value fai_rmi_install) -eq 1 ]; then - install_profile_number=$PROFILE_RMI_HUB - common_pre_install - install_rmi_hub - common_post_install - elif [ $(get_boolean_conf_value fai_wf_install) -eq 1 ]; then + no_fai_profile=0 + fi + + if [ ${fai_editor_install-0} -eq 1 ]; then + install_profile_number=$PROFILE_EDITORIAL_SERVER + install_editorial_server + no_fai_profile=0 + fi + + if [ ${fai_search_install-0} -eq 1 ]; then + install_profile_number=$PROFILE_SEARCH_SERVER + install_search_server + no_fai_profile=0 + fi + + if [ ${fai_wf_install-0} -eq 1 ]; then install_profile_number=$PROFILE_WIDGET_FRAMEWORK - common_pre_install install_widget_framework - common_post_install - elif [ $(get_boolean_conf_value fai_presentation_install) -eq 1 ]; then + no_fai_profile=0 + fi + + if [ ${fai_presentation_install-0} -eq 1 ]; then install_profile_number=$PROFILE_PRESENTATION_SERVER - common_pre_install install_presentation_server - common_post_install - elif [ $(get_boolean_conf_value fai_publication_create) -eq 1 ]; then + no_fai_profile=0 + fi + + if [ ${fai_cache_install-0} -eq 1 ]; then + install_profile_number=$PROFILE_CACHE_SERVER + install_cache_server + no_fai_profile=0 + fi + + if [ ${fai_publication_create-0} -eq 1 ]; then install_profile_number=$PROFILE_CREATE_PUBLICATION - common_pre_install create_publication - common_post_install + no_fai_profile=0 + fi + + if [ ${fai_restore_from_backup-0} -eq 1 ]; then + install_profile_number=$PROFILE_RESTORE_FROM_BACKUP + restore_from_backup + no_fai_profile=0 fi + + if [ ${fai_rmi_install-0} -eq 1 ]; then + install_profile_number=$PROFILE_RMI_HUB + install_rmi_hub + no_fai_profile=0 + fi + + + if [ ${fai_analysis_db_install-0} -eq 1 ]; then + install_profile_number=$PROFILE_ANALYSIS_DB_SERVER + install_database_server + no_fai_profile=0 + fi + + # Important that the EAE is installed after all, editor and + # presentation in case it's running on the same host as any of + # them. This because it configures the EAE ECE plugin, which the + # ECE profiles also will copy out if EAE is listed in the + # technet_download_list. + if [ ${fai_analysis_install-0} -eq 1 ]; then + install_profile_number=$PROFILE_ANALYSIS_SERVER + install_analysis_server + no_fai_profile=0 + fi + + if [ ${fai_nfs_server_install-0} -eq 1 ]; then + install_profile_number=$PROFILE_NFS_SERVER + install_nfs_server + no_fai_profile=0 + fi + + # checking for VIP profile last so that ece-install can (if so + # configured in the ece-install.con) install all dependent + # services first. + if [ ${fai_vip_install-0} -eq 1 ]; then + install_profile_number=$PROFILE_VIP_PROVIDER + install_vip_provider + no_fai_profile=0 + fi + + if [ ${fai_db_daily_backup-0} -eq 1 ]; then + install_profile_number=$PROFILE_DB_BACKUP_SERVER + install_db_backup_server + no_fai_profile=0 + fi + + if [ $no_fai_profile -eq 1 ]; then + print_and_log "No install profile selected, be sure to have one of the " + print_and_log "fai__install=1 in your $conf_file" + remove_pid_and_exit_in_error + fi + + common_post_install else read_user_input common_pre_install + case $install_profile_number in $PROFILE_ALL_IN_ONE) - install_all_in_one_environment - ;; + install_all_in_one_environment + ;; $PROFILE_CACHE_SERVER) - install_cache_server - ;; + install_cache_server + ;; $PROFILE_DB_SERVER) - install_database_server - ;; + install_database_server + ;; $PROFILE_EDITORIAL_SERVER) - install_editorial_server - ;; + install_editorial_server + ;; $PROFILE_PRESENTATION_SERVER) - install_presentation_server - ;; + install_presentation_server + ;; $PROFILE_SEARCH_SERVER) - install_search_instance - ;; + install_search_server + ;; $PROFILE_RMI_HUB) - install_rmi_hub - ;; + install_rmi_hub + ;; $PROFILE_WIDGET_FRAMEWORK) - install_widget_framework - ;; + install_widget_framework + ;; $PROFILE_CREATE_PUBLICATION) - create_publication - ;; - default) - print "You must select 1-7" - exit 1 + create_publication + ;; + $PROFILE_RESTORE_FROM_BACKUP) + restore_from_backup + ;; + $PROFILE_VIP_PROVIDER) + install_vip_provider + ;; + $PROFILE_DB_BACKUP_SERVER) + set_up_db_backup + ;; + *) + print "Invalid profile number $install_profile_number, must be 1-11" + remove_pid_and_exit_in_error ;; esac common_post_install fi - +exit 0 diff --git a/usr/sbin/outgoing-connections b/usr/sbin/outgoing-connections new file mode 100755 index 00000000..40dbdbad --- /dev/null +++ b/usr/sbin/outgoing-connections @@ -0,0 +1,40 @@ +#! /usr/bin/env bash + +## Does analysis of iptables logging outgoing connections. Here's an +## example iptables rule that logs all newly created outgoing +## connections: +## +## iptables \ +## --append OUTPUT \ +## --match state \ +## --protocol tcp \ +## --state NEW \ +## -j LOG \ +## --log-uid + +log=/var/log/syslog + +function resolve_ip() { + nslookup $1 | \ + grep 'name =' | \ + sed 's/^.*name = //g' | \ + sed 's/\.$//g' +} + +function analyze_log() { + echo 'Count IP Hostname' + grep DST= $log | \ + sed 's/.*DST=//g' | \ + cut -d' ' -f1 | \ + sort | \ + uniq -c | \ + sort -r -n | \ + sed 's/^[ ]*//g' | \ + while read line; do + read count ip <<< $line; + echo "${count} $ip $(resolve_ip $ip)" + done +} + +analyze_log | column -t + diff --git a/usr/share/check_mk/checks/indexer_running b/usr/share/check_mk/checks/indexer_running new file mode 100755 index 00000000..1cedd84a --- /dev/null +++ b/usr/share/check_mk/checks/indexer_running @@ -0,0 +1,76 @@ +#!/usr/bin/python +# -*- encoding: utf-8; py-indent-offset: 4 -*- +# +------------------------------------------------------------------+ +# | ____ _ _ __ __ _ __ | +# | / ___| |__ ___ ___| | __ | \/ | |/ / | +# | | | | '_ \ / _ \/ __| |/ / | |\/| | ' / | +# | | |___| | | | __/ (__| < | | | | . \ | +# | \____|_| |_|\___|\___|_|\_\___|_| |_|_|\_\ | +# | | +# | Copyright Mathias Kettner 2010 mk@mathias-kettner.de | +# +------------------------------------------------------------------+ +# +# This file is part of Check_MK. +# The official homepage is at http://mathias-kettner.de/check_mk. +# +# check_mk is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation in version 2. check_mk is distributed +# in the hope that it will be useful, but WITHOUT ANY WARRANTY; with- +# out even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. See the GNU General Public License for more de- +# ails. You should have received a copy of the GNU General Public +# License along with GNU Make; see the file COPYING. If not, write +# to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, +# Boston, MA 02110-1301 USA. + +# expected line is +# indexer_running +# + +import time + +def inventory_indexer_running(info): + items = [] + for line in info: + item = line[1] + items.append((item, None)) + return items + + +def check_indexer_running(item, params, info): + for line in info: + state=3 + msg="No data for " + line[1] + result=state, msg + if line[1] == item: + sample_epoch=int(line[0]) + running=line[2] + age=line[3] + filehead=line[4] + dbhead=line[5] +# comment=line[6:] + curr_epoch=int(time.time()) + if (sample_epoch + 90) < curr_epoch: + state=2 + msg="Data is not updated." + result=state, msg + return result + #msg=" ".join(str(w) for w in line[2:]) + if running == "NO": + state=2 + msg="CRITICAL. Stopped for "+ age +" min. Head in file:"+ filehead +" and DB:" + dbhead + elif running == "no": + state=1 + msg="WARNING. Stopped for "+ age +" min. Head in file:"+ filehead +" and DB:" + dbhead + else: + state=0 + msg="OK. Last index took "+ age +" min. Head in file:"+ filehead +" and DB:" + dbhead + result=state, msg + return result + +check_info['indexer_running'] = ( + check_indexer_running, + "%s", + 1, + inventory_indexer_running) diff --git a/usr/share/check_mk/checks/site_length b/usr/share/check_mk/checks/site_length new file mode 100644 index 00000000..8ef2ac36 --- /dev/null +++ b/usr/share/check_mk/checks/site_length @@ -0,0 +1,83 @@ +#!/usr/bin/python +# -*- encoding: utf-8; py-indent-offset: 4 -*- +# +------------------------------------------------------------------+ +# | ____ _ _ __ __ _ __ | +# | / ___| |__ ___ ___| | __ | \/ | |/ / | +# | | | | '_ \ / _ \/ __| |/ / | |\/| | ' / | +# | | |___| | | | __/ (__| < | | | | . \ | +# | \____|_| |_|\___|\___|_|\_\___|_| |_|_|\_\ | +# | | +# | Copyright Mathias Kettner 2010 mk@mathias-kettner.de | +# +------------------------------------------------------------------+ +# +# This file is part of Check_MK. +# The official homepage is at http://mathias-kettner.de/check_mk. +# +# check_mk is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation in version 2. check_mk is distributed +# in the hope that it will be useful, but WITHOUT ANY WARRANTY; with- +# out even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. See the GNU General Public License for more de- +# ails. You should have received a copy of the GNU General Public +# License along with GNU Make; see the file COPYING. If not, write +# to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, +# Boston, MA 02110-1301 USA. + +# expected line is +#