name: rocky-opensearch
description: OpenSearch 3.4 cluster administration on Rocky Linux 8/9 including RPM installation, kernel tuning, single-node and multi-node cluster configuration, security plugin, index and ISM policy management, snapshots, JVM heap sizing, and performance tuning. Use when installing, configuring, or administering OpenSearch clusters.

OpenSearch 3.4 Cluster Administration

Installation, configuration, security, index management, snapshots, and performance tuning for OpenSearch 3.4 on Rocky Linux 8/9.

Prerequisite: See rocky-foundation for OS detection and safety tier definitions.

Installation

RPM Repository Setup

# Import GPG key  # [CONFIRM]
rpm --import https://artifacts.opensearch.org/publickeys/opensearch.pgp

# Add repository  # [CONFIRM]
cat > /etc/yum.repos.d/opensearch-3.x.repo << 'EOF'
[opensearch-3.x]
name=OpenSearch 3.x
baseurl=https://artifacts.opensearch.org/releases/bundle/opensearch/3.x/yum
enabled=1
gpgcheck=1
gpgkey=https://artifacts.opensearch.org/publickeys/opensearch.pgp
type=rpm-md
EOF

# Install OpenSearch 3.4  # [CONFIRM]
dnf install -y opensearch-3.4.0

# Install OpenSearch Dashboards (optional)  # [CONFIRM]
cat > /etc/yum.repos.d/opensearch-dashboards-3.x.repo << 'EOF'
[opensearch-dashboards-3.x]
name=OpenSearch Dashboards 3.x
baseurl=https://artifacts.opensearch.org/releases/bundle/opensearch-dashboards/3.x/yum
enabled=1
gpgcheck=1
gpgkey=https://artifacts.opensearch.org/publickeys/opensearch.pgp
type=rpm-md
EOF

dnf install -y opensearch-dashboards-3.4.0

Kernel Tuning

# Required: vm.max_map_count  # [CONFIRM]
echo "vm.max_map_count=262144" > /etc/sysctl.d/99-opensearch.conf
sysctl -p /etc/sysctl.d/99-opensearch.conf

# Verify  # [READ-ONLY]
sysctl vm.max_map_count

# File descriptor limits  # [CONFIRM]
cat > /etc/security/limits.d/opensearch.conf << 'EOF'
opensearch soft nofile 65535
opensearch hard nofile 65535
opensearch soft nproc 4096
opensearch hard nproc 4096
opensearch soft memlock unlimited
opensearch hard memlock unlimited
EOF

# Disable swap (or reduce swappiness)  # [CONFIRM]
swapoff -a                               # Temporary
# Or permanently: comment out swap in /etc/fstab
# Or set swappiness:
echo "vm.swappiness=1" >> /etc/sysctl.d/99-opensearch.conf
sysctl -p /etc/sysctl.d/99-opensearch.conf

Systemd Service Tuning

# Override systemd limits for opensearch  # [CONFIRM]
mkdir -p /etc/systemd/system/opensearch.service.d
cat > /etc/systemd/system/opensearch.service.d/override.conf << 'EOF'
[Service]
LimitNOFILE=65535
LimitNPROC=4096
LimitMEMLOCK=infinity
EOF

systemctl daemon-reload

Cluster Configuration

Configuration Files

/etc/opensearch/
├── opensearch.yml            # Main configuration
├── jvm.options               # JVM heap and GC settings
├── jvm.options.d/            # JVM options drop-in directory
├── log4j2.properties         # Logging configuration
└── opensearch-security/      # Security plugin config
    ├── config.yml
    ├── internal_users.yml
    ├── roles.yml
    ├── roles_mapping.yml
    ├── action_groups.yml
    └── tenants.yml

Single-Node Configuration

# /etc/opensearch/opensearch.yml  # [CONFIRM]
cluster.name: my-cluster
node.name: node-1

# Network
network.host: 0.0.0.0
http.port: 9200
transport.port: 9300

# Single-node discovery
discovery.type: single-node

# Paths
path.data: /var/lib/opensearch
path.logs: /var/log/opensearch

# Security plugin (disable for development only)
# plugins.security.disabled: true

Multi-Node Configuration

# /etc/opensearch/opensearch.yml (node-1)  # [CONFIRM]
cluster.name: production-cluster
node.name: node-1

# Node roles
node.roles: [cluster_manager, data, ingest]

# Network
network.host: 10.0.0.11
http.port: 9200
transport.port: 9300

# Cluster discovery
discovery.seed_hosts:
  - 10.0.0.11
  - 10.0.0.12
  - 10.0.0.13
cluster.initial_cluster_manager_nodes:
  - node-1
  - node-2
  - node-3

# Paths
path.data: /var/lib/opensearch
path.logs: /var/log/opensearch

Node Roles

Start and Verify

# Enable and start  # [CONFIRM]
systemctl enable --now opensearch

# Check cluster health  # [READ-ONLY]
curl -sk -u admin:admin https://localhost:9200/_cluster/health?pretty
# or without security:
curl -s http://localhost:9200/_cluster/health?pretty

# Check node info  # [READ-ONLY]
curl -sk -u admin:admin https://localhost:9200/_cat/nodes?v

# Check cluster settings  # [READ-ONLY]
curl -sk -u admin:admin https://localhost:9200/_cluster/settings?include_defaults=true&flat_settings=true | python3 -m json.tool | head -50

Security Plugin

Initial Setup

# The security plugin ships with demo certificates
# For production: generate your own TLS certificates

# Security configuration directory
ls -la /etc/opensearch/opensearch-security/  # [READ-ONLY]

TLS Configuration

# /etc/opensearch/opensearch.yml -- TLS section  # [CONFIRM]
plugins.security.ssl.transport.pemcert_filepath: /etc/opensearch/certs/node.pem
plugins.security.ssl.transport.pemkey_filepath: /etc/opensearch/certs/node-key.pem
plugins.security.ssl.transport.pemtrustedcas_filepath: /etc/opensearch/certs/ca.pem
plugins.security.ssl.transport.enforce_hostname_verification: true

plugins.security.ssl.http.enabled: true
plugins.security.ssl.http.pemcert_filepath: /etc/opensearch/certs/node-http.pem
plugins.security.ssl.http.pemkey_filepath: /etc/opensearch/certs/node-http-key.pem
plugins.security.ssl.http.pemtrustedcas_filepath: /etc/opensearch/certs/ca.pem

plugins.security.authcz.admin_dn:
  - "CN=admin,OU=ops,O=myorg,C=US"
plugins.security.nodes_dn:
  - "CN=node*,OU=ops,O=myorg,C=US"

Internal Users

# /etc/opensearch/opensearch-security/internal_users.yml  # [CONFIRM]
_meta:
  type: "internalusers"
  config_version: 2

admin:
  hash: "$2y$12$..."     # Generate with: /usr/share/opensearch/plugins/opensearch-security/tools/hash.sh
  reserved: true
  backend_roles:
    - "admin"
  description: "Admin user"

kibanaserver:
  hash: "$2y$12$..."
  reserved: true
  description: "Dashboards user"

readall:
  hash: "$2y$12$..."
  reserved: false
  backend_roles:
    - "readall"
  description: "Read-only user"

Apply Security Configuration

# Hash a password  # [READ-ONLY]
/usr/share/opensearch/plugins/opensearch-security/tools/hash.sh

# Apply security config changes  # [CONFIRM]
OPENSEARCH_JAVA_HOME=/usr/share/opensearch/jdk \
/usr/share/opensearch/plugins/opensearch-security/tools/securityadmin.sh \
  -cd /etc/opensearch/opensearch-security/ \
  -icl -nhnv \
  -cacert /etc/opensearch/certs/ca.pem \
  -cert /etc/opensearch/certs/admin.pem \
  -key /etc/opensearch/certs/admin-key.pem

Index Management

# List indices  # [READ-ONLY]
curl -sk -u admin:admin https://localhost:9200/_cat/indices?v&s=index

# Create index with mappings  # [CONFIRM]
curl -sk -u admin:admin -X PUT https://localhost:9200/my-index -H 'Content-Type: application/json' -d '{
  "settings": {
    "number_of_shards": 3,
    "number_of_replicas": 1,
    "refresh_interval": "5s"
  },
  "mappings": {
    "properties": {
      "timestamp": { "type": "date" },
      "message": { "type": "text" },
      "level": { "type": "keyword" },
      "host": { "type": "keyword" }
    }
  }
}'

# Get index settings  # [READ-ONLY]
curl -sk -u admin:admin https://localhost:9200/my-index/_settings?pretty
curl -sk -u admin:admin https://localhost:9200/my-index/_mapping?pretty

# Delete index  # [DESTRUCTIVE]
curl -sk -u admin:admin -X DELETE https://localhost:9200/my-index

# Index stats  # [READ-ONLY]
curl -sk -u admin:admin https://localhost:9200/my-index/_stats?pretty
curl -sk -u admin:admin https://localhost:9200/_cat/shards?v&s=index

Index Templates

# Create index template  # [CONFIRM]
curl -sk -u admin:admin -X PUT https://localhost:9200/_index_template/logs-template -H 'Content-Type: application/json' -d '{
  "index_patterns": ["logs-*"],
  "template": {
    "settings": {
      "number_of_shards": 3,
      "number_of_replicas": 1
    },
    "mappings": {
      "properties": {
        "@timestamp": { "type": "date" },
        "message": { "type": "text" },
        "level": { "type": "keyword" }
      }
    }
  },
  "priority": 100
}'

# List templates  # [READ-ONLY]
curl -sk -u admin:admin https://localhost:9200/_index_template?pretty

ISM (Index State Management) Policies

# Create ISM policy for log rotation  # [CONFIRM]
curl -sk -u admin:admin -X PUT https://localhost:9200/_plugins/_ism/policies/log-rotation -H 'Content-Type: application/json' -d '{
  "policy": {
    "description": "Rotate logs: warm after 7 days, cold after 30 days, delete after 90 days",
    "default_state": "hot",
    "states": [
      {
        "name": "hot",
        "actions": [
          { "rollover": { "min_index_age": "7d", "min_primary_shard_size": "30gb" } }
        ],
        "transitions": [
          { "state_name": "warm", "conditions": { "min_index_age": "7d" } }
        ]
      },
      {
        "name": "warm",
        "actions": [
          { "replica_count": { "number_of_replicas": 0 } },
          { "force_merge": { "max_num_segments": 1 } }
        ],
        "transitions": [
          { "state_name": "cold", "conditions": { "min_index_age": "30d" } }
        ]
      },
      {
        "name": "cold",
        "actions": [
          { "read_only": {} }
        ],
        "transitions": [
          { "state_name": "delete", "conditions": { "min_index_age": "90d" } }
        ]
      },
      {
        "name": "delete",
        "actions": [
          { "delete": {} }
        ]
      }
    ],
    "ism_template": [
      { "index_patterns": ["logs-*"], "priority": 100 }
    ]
  }
}'

# Check ISM policy status  # [READ-ONLY]
curl -sk -u admin:admin https://localhost:9200/_plugins/_ism/explain/logs-*?pretty

Snapshots

Configure Snapshot Repository

# Create snapshot directory  # [CONFIRM]
mkdir -p /mnt/opensearch-snapshots
chown opensearch:opensearch /mnt/opensearch-snapshots

# Add to opensearch.yml  # [CONFIRM]
# path.repo: ["/mnt/opensearch-snapshots"]

# Restart after config change  # [CONFIRM]
systemctl restart opensearch

# Register repository  # [CONFIRM]
curl -sk -u admin:admin -X PUT https://localhost:9200/_snapshot/my-backup -H 'Content-Type: application/json' -d '{
  "type": "fs",
  "settings": {
    "location": "/mnt/opensearch-snapshots/my-backup"
  }
}'

# Verify repository  # [READ-ONLY]
curl -sk -u admin:admin -X POST https://localhost:9200/_snapshot/my-backup/_verify?pretty

Create and Restore Snapshots

# Create snapshot  # [CONFIRM]
curl -sk -u admin:admin -X PUT "https://localhost:9200/_snapshot/my-backup/snapshot-$(date +%Y%m%d)?wait_for_completion=true" -H 'Content-Type: application/json' -d '{
  "indices": "logs-*",
  "include_global_state": false
}'

# List snapshots  # [READ-ONLY]
curl -sk -u admin:admin https://localhost:9200/_snapshot/my-backup/_all?pretty

# Snapshot status  # [READ-ONLY]
curl -sk -u admin:admin https://localhost:9200/_snapshot/my-backup/_current?pretty

# Restore snapshot  # [DESTRUCTIVE]
curl -sk -u admin:admin -X POST https://localhost:9200/_snapshot/my-backup/snapshot-20240115/_restore -H 'Content-Type: application/json' -d '{
  "indices": "logs-2024.01.*",
  "rename_pattern": "(.+)",
  "rename_replacement": "restored-$1"
}'

# Delete old snapshot  # [DESTRUCTIVE]
curl -sk -u admin:admin -X DELETE https://localhost:9200/_snapshot/my-backup/snapshot-20240101

JVM and Performance Tuning

JVM Heap Sizing

# Set heap size (50% of available RAM, max 32GB)  # [CONFIRM]
# /etc/opensearch/jvm.options.d/heap.options
cat > /etc/opensearch/jvm.options.d/heap.options << 'EOF'
-Xms16g
-Xmx16g
EOF

# Restart required after heap change  # [CONFIRM]
systemctl restart opensearch

Rules for heap sizing:
- Set -Xms and -Xmx to the same value
- No more than 50% of available RAM (OS needs the rest for filesystem cache)
- Never exceed 32GB (compressed oops threshold)
- For dedicated nodes: min(RAM/2, 32GB)

Check JVM and Cluster Health

# Node stats  # [READ-ONLY]
curl -sk -u admin:admin https://localhost:9200/_nodes/stats/jvm?pretty
curl -sk -u admin:admin https://localhost:9200/_nodes/stats/os?pretty

# Hot threads  # [READ-ONLY]
curl -sk -u admin:admin https://localhost:9200/_nodes/hot_threads

# Pending tasks  # [READ-ONLY]
curl -sk -u admin:admin https://localhost:9200/_cluster/pending_tasks?pretty

# Allocation explanation  # [READ-ONLY]
curl -sk -u admin:admin https://localhost:9200/_cluster/allocation/explain?pretty

# Shard allocation  # [READ-ONLY]
curl -sk -u admin:admin https://localhost:9200/_cat/shards?v&s=state,index

Performance Settings

# /etc/opensearch/opensearch.yml -- Performance  # [CONFIRM]
# Thread pools (adjust based on workload)
thread_pool.write.queue_size: 1000
thread_pool.search.queue_size: 1000

# Circuit breakers
indices.breaker.total.limit: 70%
indices.breaker.fielddata.limit: 40%

# Indexing buffer
indices.memory.index_buffer_size: 15%

# Cache
indices.queries.cache.size: 15%

SELinux for OpenSearch

# Custom data directory  # [CONFIRM]
semanage fcontext -a -t var_lib_t "/data/opensearch(/.*)?"
restorecon -Rv /data/opensearch/

# Allow network binding  # [CONFIRM]
semanage port -a -t http_port_t -p tcp 9200
semanage port -a -t http_port_t -p tcp 9300

Firewall Rules

# Open ports  # [CONFIRM]
firewall-cmd --add-port=9200/tcp --permanent     # HTTP API
firewall-cmd --add-port=9300/tcp --permanent     # Transport (cluster)
firewall-cmd --add-port=5601/tcp --permanent     # Dashboards (if installed)
firewall-cmd --reload

# Restrict to specific sources  # [CONFIRM]
firewall-cmd --permanent --add-rich-rule='
  rule family="ipv4"
  source address="10.0.0.0/8"
  port protocol="tcp" port="9200"
  accept'
firewall-cmd --reload

Monitoring and Troubleshooting

# Cluster health  # [READ-ONLY]
curl -sk -u admin:admin https://localhost:9200/_cluster/health?pretty

# Unassigned shards  # [READ-ONLY]
curl -sk -u admin:admin https://localhost:9200/_cat/shards?v&h=index,shard,prirep,state,unassigned.reason&s=state

# Disk space  # [READ-ONLY]
curl -sk -u admin:admin https://localhost:9200/_cat/allocation?v

# Index sizes  # [READ-ONLY]
curl -sk -u admin:admin 'https://localhost:9200/_cat/indices?v&s=store.size:desc'

# Slow queries  # [READ-ONLY]
# Configure in opensearch.yml:
# index.search.slowlog.threshold.query.warn: 10s
# index.search.slowlog.threshold.query.info: 5s
journalctl -u opensearch | grep slowlog

When to Use This Skill

• Installing or upgrading OpenSearch
• Configuring cluster topology (single-node or multi-node)
• Managing the security plugin (users, TLS, authentication)
• Creating indices, templates, or ISM policies
• Setting up and managing snapshots
• Tuning JVM heap and performance settings
• Troubleshooting cluster health issues

Related Skills

• rocky-foundation -- OS detection, safety tiers, kernel tuning
• rocky-core-system -- systemd service management
• rocky-selinux -- SELinux contexts for OpenSearch paths/ports
• rocky-networking -- Firewall rules for cluster ports
• rocky-opentelemetry -- Sending logs/metrics to OpenSearch
• rocky-borg-backup -- Backing up OpenSearch config and snapshots