Difference between revisions of "Elasticsearch"

 cd ~
 wget https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch-1.7.6.deb
 sudo dpkg --install elasticsearch-1.7.6.deb
 # Unpacking elasticsearch (from elasticsearch-1.7.6.deb) ...
 # Setting up elasticsearch (1.7.6) ...

 sudo /usr/share/elasticsearch/bin/plugin install elasticsearch/elasticsearch-mapper-attachments/2.7.1

 cd ~
 wget https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/deb/elasticsearch/2.4.5/elasticsearch-2.4.5.deb
 # https://www.elastic.co/guide/en/elasticsearch/reference/2.4/release-notes-2.4.5.html
 sudo dpkg --install elasticsearch-2.4.5.deb
 # Unpacking elasticsearch (from elasticsearch-2.4.5.deb) ...
 # Creating elasticsearch group... OK
 # Creating elasticsearch user... OK
 # Setting up elasticsearch (2.4.5) ...

 sudo /usr/share/elasticsearch/bin/plugin install mapper-attachments # deprecated in elasticsearch 5+
 # @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
 # @     WARNING: plugin requires additional permissions     @
 # @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
 # * java.lang.RuntimePermission getClassLoader
 # * java.lang.reflect.ReflectPermission suppressAccessChecks
 # * java.security.SecurityPermission createAccessControlContext
 # * java.security.SecurityPermission insertProvider
 # * java.security.SecurityPermission insertProvider.BC
 # * java.security.SecurityPermission putProviderProperty.BC
 # See http://docs.oracle.com/javase/8/docs/technotes/guides/security/permissions.html
 # for descriptions of what these permissions allow and the associated risks.
 # 
 # Continue with installation? [y/N] y
 # Installed mapper-attachments into /usr/share/elasticsearch/plugins/mapper-attachments

// LocalSettings.php
$wgHooks['CirrusSearchMappingConfig'][] = function ( array &$config, $mappingConfigBuilder ) {
  // ... add mapping here
  $config['page']['properties']['file_attachment'] = [
    'type' => 'attachment',
    "fields" => [
      "content" => [
        "type" => "string",
        "copy_to" => ["all", "file_text"],
      ]
    ]
  ];
};

 dpkg-query -L elasticsearch # check installed files of the package
 # /etc/init.d/elasticsearch               - system service, also configuration and system variables
 # /usr/lib/sysctl.d/elasticsearch.conf    - contains system service configuration
 # /etc/elasticsearch/                     - contains elasticsearch configuration see elasticsearch.yml
 # /usr/share/elasticsearch/bin/plugin     - binary for plugin management
 # /usr/share/elasticsearch/NOTICE.txt     - documentation
 # /usr/share/elasticsearch/README.textile - documentation
 # /var/lib/elasticsearch                  - default data directory (set in /etc/init.d/elasticsearch)

 if ! [[ -d /mnt/storage/var-lib-elasticsearch ]]; then 
   sudo mkdir /mnt/storage/var-lib-elasticsearch;
   sudo mv /var/lib/elasticsearch  /var/lib/elasticsearch-bak
   cd /var/lib # make elasticsearch store data to storage device, i.e. symlink it
   sudo ln -s --force /mnt/storage/var-lib-elasticsearch elasticsearch
   sudo chown elasticsearch:elasticsearch -R /mnt/storage/var-lib-elasticsearch
   sudo chown elasticsearch:elasticsearch -R elasticsearch # set symlink
   # sudo rm --interactive /var/lib/elasticsearch-bak
 fi

 sudo vi /etc/init.d/elasticsearch # set ES_HEAP_SIZE not clear how much
 sudo vi /etc/elasticsearch/elasticsearch.yml
 # set cluster.name: biowikifarm-prod
 #   affects also name of log file in /var/log/elasticsearch/, e.g. biowikifarm-prod.log
 # set node.name: ${HOSTNAME}

ps --pid 1 # init

 # update-rc.d name-of-service defaults Start-Sequence-Number  Kill-Sequence-Number
 sudo update-rc.d elasticsearch defaults 95 10

sudo -i service elasticsearch status
sudo -i service elasticsearch start
sudo -i service elasticsearch stop

 cd /usr/share/mediawiki26/extensions-rich-features/Elastica
 sudo -u www-data /usr/local/bin/composer.phar install --no-dev
 
 cd /usr/share/mediawiki26/extensions-simple-features/Elastica
 sudo -u www-data /usr/local/bin/composer.phar install --no-dev

require_once( "$IP/extensions/Elastica/Elastica.php" );
require_once( "$IP/extensions/CirrusSearch/CirrusSearch.php" );

$wgDisableSearchUpdate = true;

 WIKI_PATH=/var/www/testwiki2/
 # WIKI_PATH=/var/www/v-species/s/
 # WIKI_PATH=/var/www/v-species/o/
 cd $WIKI_PATH
 sudo -u www-data php ./extensions/CirrusSearch/maintenance/updateSearchIndexConfig.php  --conf LocalSettings.php

# LocalSettings.php switch to false or remove $wgDisableSearchUpdate
$wgDisableSearchUpdate = false;

 WIKI_PATH=/var/www/testwiki2/
 cd $WIKI_PATH
 sudo -u www-data php ./extensions/CirrusSearch/maintenance/forceSearchIndex.php --skipLinks --indexOnSkip --conf LocalSettings.php # takes a long time
 sudo -u www-data php ./extensions/CirrusSearch/maintenance/forceSearchIndex.php --skipParse --conf LocalSettings.php

# LocalSettings.php switch Searchtype
 $wgSearchType = 'CirrusSearch';

# LocalSettings.php
$wgDisableSearchUpdate = true;
# $wgSearchType = 'CirrusSearch'; # uncomment usually switches back to default search

 WIKI_PATH=/var/www/testwiki2/
 # WIKI_PATH=/var/www/v-species/s/
 # WIKI_PATH=/var/www/v-species/o/
 cd $WIKI_PATH
 sudo -u www-data php ./extensions/CirrusSearch/maintenance/updateSearchIndexConfig.php  --conf LocalSettings.php

# LocalSettings.php switch to false or remove it:
$wgDisableSearchUpdate = false;

 WIKI_PATH=/var/www/testwiki2/
 cd $WIKI_PATH
 # 1st STEP: takes a long time (openmendia 3 to 4 days)
 sudo -u www-data php ./extensions/CirrusSearch/maintenance/forceSearchIndex.php --skipLinks --indexOnSkip --conf LocalSettings.php
   # if it got interrupted, further from a particular page-ID e.g. --fromId 180100
   sudo -u www-data php ./extensions/CirrusSearch/maintenance/forceSearchIndex.php --skipLinks --indexOnSkip --fromId 180100 --conf LocalSettings.php
 # 2nd STEP: runs faster, skip parsing the page
 sudo -u www-data php ./extensions/CirrusSearch/maintenance/forceSearchIndex.php --skipParse --conf LocalSettings.php

# LocalSettings.php
$wgDisableSearchUpdate = false;
$wgSearchType = 'CirrusSearch'; # use cirrus search again

export N_SCRIPT_FILES=10 # or whatever number you want
wiki="openmedia"
WIKI_PATH=/var/www/v-species/o
if [[ -d  $WIKI_PATH ]]; then
  cd $WIKI_PATH
  if [[ -d cirrus_scripts ]]; then sudo rm -rf cirrus_scripts; fi;
  mkdir cirrus_scripts
  if ! [[ -d cirrus_log ]]; then mkdir cirrus_log ; fi;
  pushd cirrus_scripts
  # generate smaller page sets to work through, try different numbers depending on your experience
  sudo -u www-data php $WIKI_PATH/extensions/CirrusSearch/maintenance/forceSearchIndex.php --queue --maxJobs 5000 --pauseForJobs 1000 \
      --skipLinks --indexOnSkip --buildChunks 5000  --conf $WIKI_PATH/LocalSettings.php |
      sed -e 's/$/ | tee -a cirrus_log\/'$wiki'.parse.log/' |
      split --number r/$N_SCRIPT_FILES
      # did output N_SCRIPT_FILES of FILE to xaa, xab
  # randomly sort commands within a script FILE to xaa, xab
  for script in x*; do sort --random-sort $script > $script.sh && rm $script; done
  popd
  sudo chown www-data:www-data -R cirrus_scripts
  sudo chown www-data:www-data -R cirrus_log
fi

export N_SCRIPT_FILES=10 # or whatever number you want
wiki="openmedia"
WIKI_PATH=/var/www/v-species/o
if [[ -d  $WIKI_PATH ]]; then
  cd $WIKI_PATH
  pushd cirrus_scripts
  rm *.sh # remove old scripts
  # generate smaller page sets to work through, try different numbers depending on your experience
  sudo -u www-data php $WIKI_PATH/extensions/CirrusSearch/maintenance/forceSearchIndex.php --queue --maxJobs 5000 --pauseForJobs 1000 \
      --skipParse --buildChunks 5000  --conf $WIKI_PATH/LocalSettings.php |
      sed -e 's/$/ | tee -a cirrus_log\/'$wiki'.parse.log/' |
      split --number r/$N_SCRIPT_FILES
      # did output N_SCRIPT_FILES of FILE to xaa, xab
  # randomly sort commands within a script FILE to xaa, xab
  for script in x*; do sort --random-sort $script > $script.sh && rm $script; done
  popd
  sudo chown www-data:www-data -R cirrus_scripts
fi

# list all the indices
# **indices** command provides a cross-section of each index. This information spans nodes
  curl -XGET 'http://localhost:9200/_cat/indices?v'

# show the **cluster health** and status
  curl -XGET 'http://localhost:9200/_cluster/health?pretty=true'
# show the **indices and status**
  curl -XGET 'http://localhost:9200/_cat/indices?v'
# show the **cluster state** (green: OK, yellow: not OK, red: major failure)
  curl -XGET 'http://localhost:9200/_cluster/state?pretty=true'
# _cat/recovery a view of index shard recoveries, both on-going and previously completed
  curl -XGET 'http://localhost:9200/_cat/recovery?v'
# show **pending tasks**
  curl -XGET 'http://localhost:9200/_cat/pending_tasks?v'
# **nodes**
  curl -XGET 'http://localhost:9200/_nodes?pretty=true' # massive detailed node infos
  curl -XGET 'http://localhost:9200/_nodes/stats?pretty=true'  # massive detailed node statistical infos
  curl -XGET 'http://localhost:9200/_cat/nodes?v' # _cat/nodes command shows the cluster topology
# **allocation** provides a snapshot of how shards have located around the cluster and the state of disk usage
  curl -XGET 'http://localhost:9200/_cat/allocation?v'
# **shards** command is the detailed view of what nodes contain which shards
  curl -XGET 'http://localhost:9200/_cat/shards?v'

# **delete** indices of e.g. onwiki
  curl -XDELETE 'localhost:9200/onwiki_content?pretty'
  curl -XDELETE 'localhost:9200/onwiki_content_first?pretty'