HerrHase 19 hours ago
parent 0e858cdb6f
commit 72fec19996

@ -1,2 +1,21 @@
# mariadb # MariaDB
## playbooks
configure - setting values for config.cnf
create-database - create new database, echo name
## missing tasks
### Setting Swappiness on Linux for MariaDB
/etc/sysctl.conf
```
vm.swappiness=1
```
## Sources
https://severalnines.com/blog/database-performance-tuning-mariadb/
https://github.com/VolkanSah/optimize-MySQL-MariaDB

@ -0,0 +1,9 @@
mariadb_memory_available:
- 8
- 16
- 32
- 64
mariadb_version: 12.3.3
mariadb_socket: "/var/run/mysqld/mysqld.sock"
mariadb_admin_user: 'root'

@ -0,0 +1,16 @@
#
# calculate memory of server and echo as gb
# workaround, because cli missing commands for this operation
#
# @author Björn Hase, me@tentakelfabrik.de
#
#
import sys
import math
result = float(sys.argv[1]) / 1000 / 1000
result = math.ceil(result)
result = int(result)
print (result)

@ -0,0 +1,38 @@
#
# Task: Generate Certificates
#
#
- include_tasks: mkdir.yml
#
#
#
- name: Generating CA
shell: |
openssl genrsa 4096 > /etc/mysql/ssl/ca-key.pem
openssl req -new -x509 -nodes -days 365000 -key /etc/mysql/ssl/ca-key.pem -out /etc/mysql/ssl/ca-cert.pem -subj "/CN={{ hostvars[inventory_hostname]['ansible_host'] }}-mysql-ca"
#
#
#
- name: Generating Client Certificate
shell: |
openssl req -newkey rsa:4096 -days 365000 -nodes -keyout /etc/mysql/ssl/client-key.pem -out /etc/mysql/ssl/client-req.pem -subj "/CN={{ hostvars[inventory_hostname]['ansible_host'] }}-mysql-server"
openssl rsa -in /etc/mysql/ssl/client-key.pem -out /etc/mysql/ssl/client-key.pem
openssl x509 -req -in /etc/mysql/ssl/client-req.pem -days 365000 -CA /etc/mysql/ssl/ca-cert.pem -CAkey /etc/mysql/ssl/ca-key.pem -set_serial 01 -out /etc/mysql/ssl/client-cert.pem
#
#
#
- name: Generating Server Certificate
shell: |
openssl req -newkey rsa:4096 -days 365000 -nodes -keyout /etc/mysql/ssl/server-key.pem -out /etc/mysql/ssl/server-req.pem -subj "/CN={{ hostvars[inventory_hostname]['ansible_host'] }}-mysql-server"
openssl rsa -in /etc/mysql/ssl/server-key.pem -out /etc/mysql/ssl/server-key.pem
openssl x509 -req -in /etc/mysql/ssl/server-req.pem -days 365000 -CA /etc/mysql/ssl/ca-cert.pem -CAkey /etc/mysql/ssl/ca-key.pem -set_serial 01 -out /etc/mysql/ssl/server-cert.pem
#
#
#
- name: Validate Certificates
shell: openssl verify -CAfile /etc/mysql/ssl/ca-cert.pem /etc/mysql/ssl/server-cert.pem /etc/mysql/ssl/client-cert.pem

@ -0,0 +1,9 @@
#
# Task: chown for Certificates
#
#
- name: Chown
shell: |
chown -R mysql:mysql /etc/mysql/ssl
chmod 644 /etc/mysql/ssl/*

@ -0,0 +1,83 @@
#
# Task: configure mysql
#
#
- script:
cmd: files/memory.py {{ current_memory.stdout }}
executable: /usr/bin/python3
register: mariadb_memory
delegate_to: localhost
vars:
ansible_become: false
- name: if memory not matching for available templates, end host
meta: end_host
when:
mariadb_memory_available | select('match', mariadb_memory.stdout_lines[0]) | list | length == 0
#
#
#
- name: adding to file if not set
lineinfile:
state: present
path: /etc/mysql/mariadb.cnf
line: "!include /etc/mysql/config.cnf"
regex: '^!include /etc/mysql/config.cnf'
changed_when: false
#
#
#
- name: create config, from default
template:
src: default.conf
dest: /etc/mysql/config.cnf
owner: root
group: root
mode: '0644'
#
# adding config from template
#
- name: append template from template based on memory
blockinfile:
dest: /etc/mysql/config.cnf
block: "{{ lookup('template', 'mariadb_' ~ mariadb_memory.stdout_lines[0] ~ 'gb.conf') }}"
insertbefore: "^\\[client\\]"
#
# change limits
#
- name:
blockinfile:
dest: /etc/security/limits.conf
block: |
mysql soft nofile 65535
mysql hard nofile 65535
#
#
#
- name: adding default for enviroments variables
template:
src: extension.conf
dest: /etc/systemd/system/mariadb.service.d/extension.conf
owner: root
group: root
mode: '0644'
#
#
#
- name: System Daemon Reload
shell: systemctl daemon-reload
#
#
#
- name: Restart service httpd, in all cases
service:
name: mysql
state: restarted

@ -0,0 +1,18 @@
#
# Task: create maridb
#
#
- name: generate name for database
set_fact:
mariadb_dbname: "db{{ lookup('community.general.random_string', length=16, upper=false, special=false) }}"
- name: create db
community.mysql.mysql_query:
login_db: "mysql"
login_unix_socket: "{{ mariadb_socket }}"
query: CREATE DATABASE {{ mariadb_dbname }} DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci
- name: result
debug:
msg: "dbname > {{ mariadb_dbname }}"

@ -0,0 +1,64 @@
#
# Task: Create User
#
# - create new User
# - optional input ip for remote access
# - remote ip will also add to ufw
#
- name: Input IP
pause:
prompt: "Remote IP?"
register: remote_ip
- name: Input Dbname
pause:
prompt: "Dbname?"
register: mariadb_dbname
#
# create username
#
- name: create username
set_fact:
mariadb_username: "user{{ lookup('community.general.random_string', length=16, upper=false, special=false) }}"
#
#
#
- name: create password
set_fact:
mariadb_password: "{{ lookup('ansible.builtin.password', '~/' + mariadb_username + '.txt', length=24, chars=['ascii_lowercase', 'digits']) }}"
#
# create user for localhast
#
- name: create user
community.mysql.mysql_query:
login_db: "mysql"
login_unix_socket: "{{ mariadb_socket }}"
query:
- CREATE USER {{ mariadb_username }}@'localhost' IDENTIFIED BY '{{ mariadb_password }}'
- GRANT SELECT,INSERT,UPDATE,DELETE,CREATE,ALTER,DROP ON {{ mariadb_dbname.user_input }}.* TO {{ mariadb_username }}@localhost
#
# create user for remote ip
#
- name: create user for remote
community.mysql.mysql_query:
login_db: "mysql"
login_unix_socket: "{{ mariadb_socket }}"
query:
- CREATE USER {{ mariadb_username }}@'{{ remote_ip.user_input }}' IDENTIFIED BY '{{ mariadb_password }}' REQUIRE SSL
- GRANT SELECT,INSERT,UPDATE,DELETE,CREATE,ALTER,DROP ON {{ mariadb_dbname.user_input }}.* TO {{ mariadb_username }}@'{{ remote_ip.user_input }}' REQUIRE SSL
when: remote_ip.user_input | regex_search('^[0-9]+.[0-9]+.[0-9]+.[0-9]+$')
#
# add rule in ufw for remote ip
#
- name: Allow IP for remote access
ufw:
rule: allow
from_ip: "{{ remote_ip.user_input }}"
port: 3306
when: remote_ip.user_input | regex_search('^[0-9]+.[0-9]+.[0-9]+.[0-9]+$')

@ -0,0 +1,15 @@
#
# Task: drop database
#
#
- name: Input name of Database
pause:
prompt: "Dbname?"
register: mariadb_dbname
- name: Drop Database
community.mysql.mysql_query:
login_db: "mysql"
login_unix_socket: "{{ mariadb_socket }}"
query: DROP DATABASE {{ mariadb_dbname.user_input }};

@ -0,0 +1,34 @@
#
# Task: export
#
# @author Björn Hase <herrhase@node001.net>
#
- name: name of database for Export
pause:
prompt: "Dbname?"
register: mariadb_dbname
- name: locale-Path for SQL-file
pause:
prompt: "Locale Path?"
register: mariadb_sql_path
- name: dump SQL from Db to file
community.mysql.mysql_db:
state: dump
name: "{{ mariadb_dbname.user_input }}"
login_user: "{{ mariadb_admin_user }}"
login_unix_socket: "{{ mariadb_socket }}"
target: "/tmp/{{ mariadb_dbname.user_input }}.sql"
- name: copy SQL-file from Db-Server to locale
fetch:
src: "/tmp/{{ mariadb_dbname.user_input }}.sql"
dest: "{{ mariadb_sql_path.user_input }}"
when: mariadb_sql_path.user_input
- name: Delete SQL-file from Db-Server
file:
path: "/tmp/{{ mariadb_dbname.user_input }}.sql"
state: absent

@ -0,0 +1,35 @@
#
# Task: import
#
# @author Björn Hase, <herrhase@node001.net>
#
- name: name of database for Import
pause:
prompt: "Dbname?"
register: mariadb_dbname
- name: locale-Path for SQL-file
pause:
prompt: "Locale Path for SQL-File?"
register: mariadb_sql_path
- name: copy SQL-file to Db-Server
copy:
src: "{{ item }}"
dest: "/tmp/{{ mariadb_dbname.user_input }}.sql"
with_items: "{{ mariadb_sql_path.user_input }}"
when: mariadb_sql_path.user_input
- name: import SQL-file
community.mysql.mysql_db:
state: import
name: "{{ mariadb_dbname.user_input }}"
login_user: "{{ mariadb_admin_user }}"
login_unix_socket: "{{ mariadb_socket }}"
target: "/tmp/{{ mariadb_dbname.user_input }}.sql"
- name: delete SQL-file from Db-Server
file:
path: "/tmp/{{ mariadb_dbname.user_input }}.sql"
state: absent

@ -0,0 +1,114 @@
#
# task: install maraidb-client
#
# author Björn Hase, <herrhase@node001.net>
#
#
#
# input ip of mariadb svser
#
- name: Input of DB-Server IP
pause:
prompt: "IP Mariadb Server?"
register: mariadb_remote_server
- name: create mysql group
group:
name: mysql
state: present
- name: add mysql user
user:
name: mysql
system: true
group: mysql
create_home: false
#
# add user to mysql
#
- name: mariadb / add current user to mysql
user:
name: "{{ ansible_user }}"
groups: mysql
append: yes
#
# check for
#
- name: Create the file, if it doesnt exist already
stat:
path: /etc/apt/sources.list.d/mariadb.list
register: list
#
#
#
- name: mariadb / prepare install
shell: 'curl -LsS https://r.mariadb.com/downloads/mariadb_repo_setup | sudo bash -s -- --mariadb-server-version="mariadb-{{ mariadb_version }}"'
when: not list.stat.exists
#
#
#
- name: mariadb / install
apt:
update_cache: yes
name:
- mariadb-client
when: not list.stat.exists
#
#
#
- include_tasks: mkdir.yml
#
# getting certificates & change permissions
#
- name: rsync certificates
shell: "sshpass -p '{{ hostvars[mariadb_remote_server.user_input]['ansible_password'] }}' rsync -rv -e 'ssh -o PubkeyAuthentication=no -o StrictHostKeyChecking=no' {{ hostvars[mariadb_remote_server.user_input]['ansible_user'] }}@{{ hostvars[mariadb_remote_server.user_input]['ansible_host'] }}:/etc/mysql/ssl/{{ item }} /etc/mysql/ssl/{{ item }}"
with_items:
- "ca-cert.pem"
- "client-cert.pem"
- "client-key.pem"
- file:
path: /etc/mysql/ssl/{{ item }}
owner: mysql
group: mysql
mode: '0640'
with_items:
- "ca-cert.pem"
- "client-cert.pem"
- "client-key.pem"
#
# Check for include
#
- name: check for include, if not set add to file
lineinfile:
state: absent
path: /etc/mysql/mariadb.cnf
regex: '^!include /etc/mysql/config.cnf'
changed_when: false
register: mariadb_line_check
- name: adding to file if not set
lineinfile:
state: present
path: /etc/mysql/mariadb.cnf
line: "!include /etc/mysql/config.cnf"
when: mariadb_line_check.found == 0
#
#
#
- name: create config, from default
template:
src: client.conf
dest: /etc/mysql/config.cnf
owner: root
group: root
mode: '0644'

@ -0,0 +1,120 @@
#
# task: install maraidb-server
#
# author Björn Hase, <herrhase@node001.net>
#
#
#
# check if maridb is already installed
#
- shell: dpkg -l | grep -e mariadb-server | cat
register: mariadb_installed
- name: if mariadb is already installed, end host
meta: end_host
when:
mariadb_installed.stdout | length > 0
#
# adding list in sources
#
- name: create the file, if it doesnt exist already
stat:
path: /etc/apt/sources.list.d/mariadb.list
register: list
#
# getting mariadb
#
- name: prepare install
shell: 'curl -LsS https://r.mariadb.com/downloads/mariadb_repo_setup | sudo bash -s -- --mariadb-server-version="mariadb-{{ mariadb_version }}"'
when: not list.stat.exists
#
# adding packages
#
- name: mariadb / install
apt:
update_cache: yes
name:
- mariadb-server
- mariadb-client
- python3-mysqldb
#
# add user to mysql
#
- name: mariadb / add current user to mysql
user:
name: "{{ ansible_user }}"
groups: mysql
append: yes
#
# getting anonymous users and remove them
#
- name: Get list of hosts for the anonymous user.
community.mysql.mysql_query:
login_db: "mysql"
login_unix_socket: "{{ mariadb_socket }}"
query: SELECT Host FROM mysql.user WHERE User = %s
positional_args:
- ''
register: mariadb_anonymous_hosts
- name: Remove anonymous MariaDB users.
community.mysql.mysql_query:
login_db: "mysql"
login_unix_socket: "{{ mariadb_socket }}"
query: DELETE FROM mysql.user WHERE User=%s AND Host=%s
positional_args:
- ''
- "{{ item.Host }}"
loop: "{{ mariadb_anonymous_hosts.query_result | first }}"
when:
- not ansible_check_mode
- mariadb_anonymous_hosts.rowcount[0] != 0
#
# getting root users and remove remote root logins
#
- name: Get list of hosts for the root user.
community.mysql.mysql_query:
login_db: "mysql"
login_unix_socket: "{{ mariadb_socket }}"
query: SELECT Host FROM mysql.user WHERE User = %s ORDER BY (Host='localhost') ASC
positional_args:
- "{{ mariadb_admin_user }}"
register: mariadb_root_hosts
- name: Disallow root login remotely
community.mysql.mysql_user:
name: "{{ mariadb_admin_user }}"
login_unix_socket: "{{ mariadb_socket }}"
host: "{{ item.Host }}"
priv: "*.*:ALL,GRANT"
state: absent
loop: '{{ mariadb_root_hosts.query_result[0] | rejectattr("Host", "in", "localhost,::1,127.0.0.1") | list }}'
when: not ansible_check_mode
#
# remove all test databses
#
- name: Remove MariaDB test database.
community.mysql.mysql_db:
name: test
state: absent
login_unix_socket: "{{ mariadb_socket }}"
#
# Certificates
#
- include_tasks: certificates.yml
- include_tasks: chown.yml
#
# Configure
#
- include_tasks: configure.yml

@ -0,0 +1,7 @@
#
#
#
- name: create Directory for Certificates
file:
path: /etc/mysql/ssl
state: directory

@ -0,0 +1,40 @@
#
# Task: remove user
#
# - remove user
# - remove remote access
# - remove ip from ufw
#
- name: Input Username
pause:
prompt: "Username?"
register: mariadb_username
- name: Search for username
community.mysql.mysql_query:
login_db: "mysql"
login_unix_socket: "{{ mariadb_socket }}"
query: SELECT Host FROM mysql.user WHERE User = '{{ mariadb_username.user_input }}'
register: mariadb_user_hosts
#
# create user for localhast
#
- name: Drop User
community.mysql.mysql_query:
login_db: "mysql"
login_unix_socket: "{{ mariadb_socket }}"
query: DROP USER '{{ mariadb_username.user_input }}'@'{{ item.Host }}'
loop: "{{ mariadb_user_hosts.query_result | flatten }}"
#
# add rule in ufw for remote ip
#
- name: Remove IP for remote access
ufw:
delete: true
rule: allow
from_ip: "{{ item.Host }}"
port: 3306
loop: "{{ mariadb_user_hosts.query_result | flatten | selectattr('Host', 'match', '^[0-9]+.[0-9]+.[0-9]+.[0-9]+$') }}"

@ -0,0 +1,4 @@
[client]
ssl-ca=/etc/mysql/ssl/ca-cert.pem
ssl-cert=/etc/mysql/ssl/client-cert.pem
ssl-key=/etc/mysql/ssl/client-key.pem

@ -0,0 +1,33 @@
[mysqld]
bind-address = 0.0.0.0
ssl-ca=/etc/mysql/ssl/ca-cert.pem
ssl-cert=/etc/mysql/ssl/server-cert.pem
ssl-key=/etc/mysql/ssl/server-key.pem
tls_version = TLSv1.2,TLSv1.3
# Slow query log settings
# - only for debug
# slow_query_log = 1
# slow_query_log_file = /var/log/mysql/mysql-slow.log
# long_query_time = 2
# Replication settings
server-id = 1
log_bin = /var/log/mysql/mysql-bin.log
binlog_format = mixed
slave_connections_needed_for_purge = 0
# Additional settings
log_bin_trust_function_creators = 1
log_error = /var/log/mysql/error.log
expire_logs_days = 10
max_binlog_size = 100M
#max_binlog_total_size = 6000M
skip_name_resolve = 1
[client]
ssl-ca=/etc/mysql/ssl/ca-cert.pem
ssl-cert=/etc/mysql/ssl/client-cert.pem
ssl-key=/etc/mysql/ssl/client-key.pem

@ -0,0 +1,6 @@
[Service]
Environment="MYSQLD_OPTS="
Environment="_WSREP_NEW_CLUSTER="
[Service]
LimitNOFILE=infinity

@ -0,0 +1,54 @@
# Performance settings
default_storage_engine = InnoDB
# innobdb
innodb_buffer_pool_instances = 4
# 75% of ram
innodb_buffer_pool_size = 12G
innodb_io_capacity = 3000
innodb_io_capacity_max = 6000
innodb_read_io_threads = 4
innodb_write_io_threads = 4
innodb_file_per_table = 1
# Flush and Logging Settings
innodb_flush_log_at_trx_commit = 2
innodb_flush_method = O_DIRECT
sync_binlog = 100
innodb_log_file_size = 512M
innodb_log_buffer_size = 32M
# Thread Settings
thread_handling = pool-of-threads
thread_pool_size = 8
thread_cache_size = 64
# Table Cache Settings
table_open_cache = 8192
table_definition_cache = 4096
table_open_cache_instances = 4
# Query Cache
query_cache_type = 0
query_cache_size = 0
# Memory
sort_buffer_size = 2M
tmp_table_size = 256M
max_heap_table_size = 256M
# Connection settings
max_connections = 500
# Additional Optimizations
innodb_change_buffering = all
innodb_checksum_algorithm = crc32
innodb_log_compressed_pages = OFF
max_allowed_packet = 64M
# Performance Schema
performance_schema = ON
[mysqld_safe]
open_files_limit=65535

@ -0,0 +1,54 @@
# Performance settings
default_storage_engine = InnoDB
# innobdb
innodb_buffer_pool_instances = 4
# 75% of ram
innodb_buffer_pool_size = 24G
innodb_io_capacity = 6000
innodb_io_capacity_max = 6000
innodb_read_io_threads = 8
innodb_write_io_threads = 8
innodb_file_per_table = 1
# Flush and Logging Settings
innodb_flush_log_at_trx_commit = 2
innodb_flush_method = O_DIRECT
sync_binlog = 100
innodb_log_file_size = 512M
innodb_log_buffer_size = 32M
# Thread Settings
thread_handling = pool-of-threads
thread_pool_size = 8
thread_cache_size = 64
# Table Cache Settings
table_open_cache = 8192
table_definition_cache = 4096
table_open_cache_instances = 4
# Query Cache
query_cache_type = 0
query_cache_size = 0
# Memory
sort_buffer_size = 2M
tmp_table_size = 256M
max_heap_table_size = 256M
# Connection settings
max_connections = 500
# Additional Optimizations
innodb_change_buffering = all
innodb_checksum_algorithm = crc32
innodb_log_compressed_pages = OFF
max_allowed_packet = 64M
# Performance Schema
performance_schema = ON
[mysqld_safe]
open_files_limit=65535

@ -0,0 +1,10 @@
# Performance settings
innodb_buffer_pool_size = 32G
innodb_log_file_size = 2048M
thread_cache_size = 64
query_cache_size = 512M
query_cache_type = 1
table_open_cache = 16384
# Connection settings
max_connections = 2000

@ -0,0 +1,10 @@
# Performance settings
innodb_buffer_pool_size = 4G
innodb_log_file_size = 256M
thread_cache_size = 8
query_cache_size = 64M
query_cache_type = 1
table_open_cache = 2048
# Connection settings
max_connections = 250
Loading…
Cancel
Save