Update ElasticSearch - Revision history

Scollazo: /* ElasticSearch update */

2019-03-01T02:37:00Z

ElasticSearch update

Scollazo: /* ElasticSearch update */

2019-03-01T02:14:14Z

ElasticSearch update

Scollazo: /* ElasticSearch update */

2019-03-01T02:13:03Z

ElasticSearch update

Mamedin: /* ElasticSearch update */

2019-02-27T17:20:04Z

ElasticSearch update

Mamedin: /* ElasticSearch update */

2019-02-27T17:16:15Z

ElasticSearch update

Scollazo: /* ElasticSearch update */

2019-02-27T00:25:44Z

ElasticSearch update

Scollazo: /* ElasticSearch update */

2019-02-27T00:02:53Z

ElasticSearch update

Scollazo: /* ElasticSearch update */

2019-02-19T18:49:22Z

ElasticSearch update

Scollazo: /* ElasticSearch update */

2019-02-19T18:25:57Z

ElasticSearch update

Scollazo: /* ElasticSearch update */

2019-02-19T18:23:29Z

ElasticSearch update

@@ Line 35: / Line 35: @@
 . Download a temporary copy of elasticsearch (same version than intalled)
-   wget https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch-1.7.5.tar.gz
+   wget https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch-1.7.6.tar.gz
 . Uncompress the downloaded tar.gz and go into it
-   tar zxvf elasticsearch-1.7.5.tar.gz
+   tar zxvf elasticsearch-1.7.6.tar.gz
-   cd elasticsearch-1.7.5
+   cd elasticsearch-1.7.6
@@ Line 60: / Line 60: @@
    curl -X GET "localhost:9500/_cat/indices?v"
-. Remove elasticsearch 1.7.5 and it's files
+. Remove elasticsearch 1.7.6 and it's files
    Redhat: sudo yum remove elasticsearch
    Ubuntu: sudo apt-get remove --purge elasticsearch
-   sudo mv /var/lib/elasticsearch /var/lib/elasticsearch-1.7.5
+   sudo mv /var/lib/elasticsearch /var/lib/elasticsearch-1.7.6
-   sudo mv /etc/elasticsearch /etc/elasticsearch-1.7.5
+   sudo mv /etc/elasticsearch /etc/elasticsearch-1.7.6
 . Upgrade archivematica and install elasticsearch 6
@@ Line 84: / Line 84: @@
          || (echo 'Environment file not found'; exit 1)
    cd /usr/share/archivematica/dashboard
-   /usr/share/archivematica/virtualenvs/archivematica-dashboard/bin/python manage.py reindex_from_remote_cluster -t 60 -s 10 http://localhost:9500
+   /usr/share/archivematica/virtualenvs/archivematica-dashboard/bin/python manage.py reindex_from_remote_cluster -t 60 -s 1 http://localhost:9500
    ";
@@ Line 103: / Line 103: @@
    yellow open   aipfiles      clX1gcqnT9CTdwTm8jeatw   5   1      24700            0     70.6mb         70.6mb
-You should have 4 indexes now, and the sum of the aips and aipfiles docs.count column should be equal to the doc.count for the aip index in ElasticSearch 1.7.5. The same happens with the transfers/transferfiles indexes.
+You should have 4 indexes now, and the sum of the aips and aipfiles docs.count column should be equal to the doc.count for the aip index in ElasticSearch 1.7.6. The same happens with the transfers/transferfiles indexes.
 . Stop the temporary elasticsearch service
-   kill $(cat ~/elasticsearch-1.7.5/elastic-tmp.pid)
+   kill $(cat ~/elasticsearch-1.7.6/elastic-tmp.pid)
 . Restart all archivematica services

@@ Line 54: / Line 54: @@
 . Launch the temporary elasticsearch instance
-   ES_JAVA_OPTS="-Xms2g -Xmx2g" ./bin/elasticsearch -d -p elastic-tmp.pid -Des.http.port=9500
+   ES_JAVA_OPTS="-Xms2g -Xmx2g" ./bin/elasticsearch -d -p elastic-tmp.pid -Des.http.port=9500  -Des.discovery.zen.ping.multicast.enabled=false
 . Verify that the indexes were properly copied. Use the port specified in the previous command:

@@ Line 111: / Line 111: @@
 . Restart all archivematica services
-   service archivematica-mcp-server restart
+   sudo service archivematica-mcp-server restart
-   service archivematica-mcp-client restart
+   sudo service archivematica-mcp-client restart
-   service archivematica-dashboard restart
+   sudo service archivematica-dashboard restart
-   service archivematica-storage-service restart
+   sudo service archivematica-storage-service restart

@@ Line 4: / Line 4: @@
    sudo service elasticsearch stop
-   sudo tar cfz var_lib_elasticsearch_$(date).tgz  /var/lib/elasticsearch
+   sudo tar cfz var_lib_elasticsearch_$(date +%y%m%d).tgz  /var/lib/elasticsearch
    sudo service elasticsearch start

@@ Line 1: / Line 1: @@
 = ElasticSearch update =
+. Check that ES is up and running, and note the results:
    curl -X GET "localhost:9200/_cat/indices?v"
@@ Line 11: / Line 14: @@
    yellow open   transfers   5   1          0            0       720b           720b
-Check also if there are any configured snapshot. If so, copy them to other place and remove them
+Check also if there are any configured snapshot:
-−
    curl -XGET "localhost:9200/_snapshot/_all?pretty"
    curl -XDELETE "localhost:9200/_snapshot/backup-repo?pretty"
+. Stop elasticsearch service
    sudo service elasticsearch stop
 . Check wich elasticsearch version is installed
-   Redhat/Centos: rpm -qa | grep elasticsearch
+   Redhat/Centos: rpm -q elasticsearch
-   Ubuntu: dpkg -l | grep elasticsearch
+   Ubuntu: dpkg -l elasticsearch
 . Download a temporary copy of elasticsearch (same version than intalled)
    wget https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch-1.7.5.tar.gz
 . Uncompress the downloaded tar.gz and go into it
    tar zxvf elasticsearch-1.7.5.tar.gz
@@ Line 37: / Line 44: @@
 . Copy data from system wide installed elasticsearch, ot the temporary copy
    sudo cp /var/lib/elasticsearch data -rf
 . Adjust file permissions
    sudo chown <your user>:<your group> data -R
 . Launch the temporary elasticsearch instance
    ES_JAVA_OPTS="-Xms2g -Xmx2g" ./bin/elasticsearch -d -p elastic-tmp.pid -Des.http.port=9500
 . Verify that the indexes were properly copied. Use the port specified in the previous command:
    curl -X GET "localhost:9500/_cat/indices?v"
 . Remove elasticsearch 1.7.5 and it's files
    Redhat: sudo yum remove elasticsearch
@@ Line 60: / Line 67: @@
    sudo mv /etc/elasticsearch /etc/elasticsearch-1.7.5
 . Upgrade archivematica and install elasticsearch 6
    ansible-playbook -i hosts singlenode.yml --tags=elasticsearch,archivematica-src
 . Configure ElasticSearch 6 to handle reindex from the temporary es
    echo 'reindex.remote.whitelist: localhost:9500' | sudo tee -a /etc/elasticsearch/elasticsearch.yml
    sudo service elasticsearch restart
 . Migrate the indexes
    sudo -u archivematica bash -c " \
@@ Line 87: / Line 94: @@
 . Verify that the new indexes were created and populated:
    curl -X GET "localhost:9200/_cat/indices?v"
@@ Line 98: / Line 105: @@
 You should have 4 indexes now, and the sum of the aips and aipfiles docs.count column should be equal to the doc.count for the aip index in ElasticSearch 1.7.5. The same happens with the transfers/transferfiles indexes.
 . Stop the temporary elasticsearch service
    kill $(cat ~/elasticsearch-1.7.5/elastic-tmp.pid)
 . Restart all archivematica services
    service archivematica-mcp-server restart
    service archivematica-mcp-client restart
    service archivematica-storage-service restart

@@ Line 77: / Line 77: @@
          || (echo 'Environment file not found'; exit 1)
    cd /usr/share/archivematica/dashboard
-   /usr/share/archivematica/virtualenvs/archivematica-dashboard/bin/python manage.py reindex_from_remote_cluster -t 60 http://localhost:9500
+   /usr/share/archivematica/virtualenvs/archivematica-dashboard/bin/python manage.py reindex_from_remote_cluster -t 60 -s 10 http://localhost:9500
    ";
@@ Line 83: / Line 83: @@
    All reindex requests ended successfully!

← Older revision		Revision as of 00:02, 27 February 2019
Line 10:		Line 10:
	yellow open aips 5 1 24816 1 175.4mb 175.4mb		yellow open aips 5 1 24816 1 175.4mb 175.4mb
	yellow open transfers 5 1 0 0 720b 720b		yellow open transfers 5 1 0 0 720b 720b
		+
		+	Check also if there are any configured snapshot. If so, copy them to other place and remove them
		+
		+	curl -XGET "localhost:9200/_snapshot/_all?pretty"
		+	curl -XDELETE "localhost:9200/_snapshot/backup-repo?pretty"

@@ Line 8: / Line 8: @@
    curl -X GET "localhost:9200/_cat/indices?v"
    health status index     pri rep docs.count docs.deleted store.size pri.store.size
-   yellow open   aips        5   1         11            0    267.9kb        267.9kb
+   yellow open   aips        5   1      24816            1    175.4mb        175.4mb
-   yellow open   transfers   5   1         24            0     35.2kb         35.2kb
+   yellow open   transfers   5   1          0            0       720b           720b
 . Stop elasticsearch service
@@ Line 41: / Line 42: @@
 . Launch the temporary elasticsearch instance
-   ./bin/elasticsearch -d -p elastic-tmp.pid -Des.http.port=9500
+   ES_JAVA_OPTS="-Xms2g -Xmx2g" ./bin/elasticsearch -d -p elastic-tmp.pid -Des.http.port=9500
 . Verify that the indexes were properly copied. Use the port specified in the previous command:
@@ Line 71: / Line 72: @@
          || (echo 'Environment file not found'; exit 1)
    cd /usr/share/archivematica/dashboard
-   /usr/share/archivematica/virtualenvs/archivematica-dashboard/bin/python manage.py reindex_from_remote_cluster http://localhost:9500
+   /usr/share/archivematica/virtualenvs/archivematica-dashboard/bin/python manage.py reindex_from_remote_cluster -t 60 http://localhost:9500
    ";
@@ Line 80: / Line 81: @@
 . Verify that the new indexes were created and populated:
-−
    curl -X GET "localhost:9200/_cat/indices?v"
    health status index         uuid                   pri rep docs.count docs.deleted store.size pri.store.size
-   yellow open   aips          IL7lbAZFR9adKg-GpgepTw   5   1          1            0    135.8kb        135.8kb
+   yellow open   transferfiles jlu2d2yZQpWwpKKT1ZzMAA   5   1          0            0      1.1kb          1.1kb
-   yellow open   transferfiles hlG1GBwIRNCi1rI74LhQpQ   5   1         23            0     49.1kb         49.1kb
+   yellow open   aips          VAvONAByRBuVcWBYhlhUmw   5   1        116            0     66.9mb         66.9mb
-   yellow open   transfers     ebRiOvutR_e7N4dNGoNq4w   5   1          1            0        6kb            6kb
+   yellow open   transfers     OhfLVBzuRqCNUYRCSSTidg   5   1          0            0      1.1kb          1.1kb
-   yellow open   aipfiles      2QFnLdr7Txi2qgG2C7-eJw   5   1         10            0    158.7kb        158.7kb
+   yellow open   aipfiles      clX1gcqnT9CTdwTm8jeatw   5   1      24700            0     70.6mb         70.6mb
 You should have 4 indexes now, and the sum of the aips and aipfiles docs.count column should be equal to the doc.count for the aip index in ElasticSearch 1.7.5. The same happens with the transfers/transferfiles indexes.