Migrating Chef CouchDB to Multi-Master CouchDB

* assuming you are using ubuntu/debian
chef-server =
couchdb01 =
couchdb02 =

Enable chef-server couchdb to listen on all interfaces

root@chefserver:~# sed -i bak s/bind_address = = /etc/couchdb/default.ini
root@chefserver:~# /etc/init.d/couchdb restart

Install CouchDB on couchdb01/couchdb02 and set to listen on all interfaces

root@couchdb01:~# apt-get -y install couchdb
root@couchdb01:~# /etc/init.d/couchdb stop
root@couchdb01:~# sed -i bak s/bind_address = = /etc/couchdb/default.ini
root@couchdb01:~# /etc/init.d/couchdb start

Create the empty chef table on couchdb01/couchdb02

root@couchdb01:~# curl -X PUT http://localhost:5984/chef

root@couchdb02:~# curl -X PUT http://localhost:5984/chef

Push the chef table from chef-server to couchdb01/02 and enable a continuous replication

To Couchdb02

root@chef-server:/var/lib/couchdb# curl -X POST http://localhost:5984/_replicate -H "Content-Type: application/json" -d '{"source":"chef","target":"","continuous":true}'

To Couchdb01

root@chef-server:~# curl -X POST http://localhost:5984/_replicate -H "Content-Type: application/json" -d '{"source":"chef","target":"","continuous":true}'

Setup the multi-master replication for couchdb01/02

Enable continuous replication FROM couchdb01 to couchdb02

root@couchdb01:/var/lib/couchdb# curl -X POST http://localhost:5984/_replicate -H "Content-Type: application/json" -d '{"source":"chef","target":"","continuous":true}'

Enable continuous replication FROM couchdb02 to couchdb01

root@couchdb02:/var/lib/couchdb# curl -X POST http://localhost:5984/_replicate -H "Content-Type: application/json" -d '{"source":"chef","target":"","continuous":true}'

Install Apache and generate config on Chef Server

root@chefserver:~# apt-get -y install apache2
root@chefserver:~# mkdir -p /usr/share/chef-server/public
root@chefserver:~# for i in rewrite proxy status proxy_http proxy_balancer headers ; do a2enmod $i ; done
root@chefserver:~# cd /etc/apache2/sites-available
root@chefserver:~# echo "Listen 5984" | tee -a chef_couchdb_loadbalancer
root@chefserver:~# echo '<VirtualHost *:5984>' | tee -a chef_couchdb_loadbalancer
root@chefserver:~# MYHOST=$(hostname -f)
root@chefserver:~# echo "ServerName ${MYHOST}-couchdb" |tee -a chef_couchdb_loadbalancer
root@chefserver:~# cat>>chef_couchdb_loadbalancer<<EOF
<Proxy balancer://couchlb>
ProxyPass / balancer://couchlb
ProxyPassReverse / balancer://couchlb
DocumentRoot /usr/share/chef-server/public
LogLevel info
ErrorLog /var/log/chef/chef_couchdb_apache2-error.log
CustomLog /var/log/chef/chef_couchdb_apache2-access.log combined
RewriteEngine On
RewriteRule ^/(.*)$ balancer://couchlb%{REQUEST_URI} [P,QSA,L]

Stop Couchdb on Chef-Server

*This will stop the continuous replication to couchdb01/couchdb02

root@chefserver:~# /etc/init.d/couchdb stop

Start Apache load balancer on Chef-Server

root@chefserver:~# a2ensite chef_couchdb_loadbalancer
root@chefserver:~# /etc/init.d/apache2 restart

Test your couchdb balancer:


You probably also want to edit your init scripts on couchdb01/02 to automatically restart the continuous replication upon start or restart
The replication does not persist after you stop a couchdb instance unless you explicitly issue the command again


Chef Server – Threading Merb – Chef Server API Service

This will only make your api system faster if you have enough cpus to support it.
It should be 1 merb worker per core ( * i think * )

Install Apache and enable the necessary mods

$ apt-get -y install apache2
$ mkdir -p /usr/share/chef-server/public
$ for i in rewrite proxy status proxy_http proxy_balancer headers ; do a2enmod $i ; done

Stop Chef-Server

$ /etc/init.d/chef-server stop

Edit your Chef configuration file ( Replace Worker Threads and Port Numbers as needed )

$ sed -i s/PORT=4000/PORT=5000/g /etc/default/chef-server
$ echo "WORKERTHREADS=4" | tee -a /etc/default/chef-server

Edit the Chef init script ( Back it up first )

$ cp /etc/init.d/chef-server /etc/init.d/chef-server.original
$ sed -i '35s/DAEMON_OPTS="-p/DAEMON_OPTS="-c $WORKERTHREADS -p/g' /etc/init.d/chef-server
$ sed -i '42s/(ps/#(ps/g' /etc/init.d/chef-server
$ sed -i '/#(ps/ i (ps -fp $pid | egrep -q "merb.*( chef-server .*api.* spawner|worker .* $PORT)") || return 1' /etc/init.d/chef-server

Run diff to see the init script differences

$ diff /etc/init.d/chef-server /etc/init.d/chef-server.original
< (ps -fp $pid | egrep -q "merb.*( chef-server .*api.* spawner|worker .* $PORT)") || return 1
<   #(ps -fp $pid | egrep -q "merb.*(merb : master|worker.*$PORT)") || return 1 --- >   (ps -fp $pid | egrep -q "merb.*(merb : master|worker.*$PORT)") || return 1

Get variables from your Chef config for apache config construction

$ THREADSCT=$(grep "WORKERTHREADS" /etc/default/chef-server |awk -F"=" '{print $2}')
$ PORTPREFIX=$(grep "PORT" /etc/default/chef-server |awk -F"=" '{print $2}'| sed -e s/[0-9][0-9]$//g )
$ MYHOST=$(hostname -f)

Generate Your Apache Config – Generate the Load Balance Members

$ cd /etc/apache2/sites-available
$ echo "Listen 4000" |tee -a chef_loadbalancer
$ echo '<VirtualHost *:4000>' |tee -a chef_loadbalancer
$ echo "ServerName $MYHOST" |tee -a chef_loadbalancer
$ echo "" |tee -a chef_loadbalancer
$ echo "<Proxy balancer://cheflb>" | tee -a chef_loadbalancer
$ seq -w 00 $NEWCOUNT | while read i ; do echo "BalancerMember${PORTPREFIX}${i}" |tee -a chef_loadbalancer ; done

Append your Apache config with the rest of the relevant information

$ cat>>chef_loadbalancer<<EOF
ProxyPass / balancer://cheflb
ProxyPassReverse / balancer://cheflb
DocumentRoot /usr/share/chef-server/public
LogLevel info
ErrorLog /var/log/chef/chef_server_apache2-error.log
CustomLog /var/log/chef/chef_server_apache2-access.log combined
RewriteEngine On
RewriteRule ^/(.*)$ balancer://cheflb%{REQUEST_URI} [P,QSA,L]

Enable your new apache config and start Apache and Chef-Server

$ a2ensite chef_loadbalancer
$ /etc/init.d/apache2 restart
$ /etc/init.d/chef-server start

Test your chef server

$ time knife node list
$ time knife role list


Much of this information was stolen from : http://mrmiller.nonesensedomains.com/2010/06/15/chef-performance-tuning-part-1/
If this technique is outdated please make me aware of it or if my apache configuration is awful ( which i’m sure it is, i’m just too lazy to improve it )